From 90dfd76418c8efeab34360911af419f4e7ceb099 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Wed, 27 Sep 2023 16:07:50 +0530 Subject: [PATCH 01/53] Better Posterior Formulation --- src/BPINN_ode.jl | 1 + src/advancedHMC_MCMC.jl | 213 ++- test/BPINN_Tests.jl | 4014 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 4210 insertions(+), 18 deletions(-) diff --git a/src/BPINN_ode.jl b/src/BPINN_ode.jl index f79f5208f2..f9a68b8917 100644 --- a/src/BPINN_ode.jl +++ b/src/BPINN_ode.jl @@ -284,6 +284,7 @@ function DiffEqBase.__solve(prob::DiffEqBase.ODEProblem, push!(ensemblecurves, ensemblecurve) end + # estimated using all samples nnparams = length(θinit) estimnnparams = [Particles(reduce(hcat, samples)[i, :]) for i in 1:nnparams] diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index 9bd4243cf6..e6b1f24faa 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -80,6 +80,7 @@ end function LogDensityProblems.logdensity(Tar::LogTargetDensity, θ) return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) + # + L2loss2(Tar, θ) end LogDensityProblems.dimension(Tar::LogTargetDensity) = Tar.dim @@ -88,25 +89,221 @@ function LogDensityProblems.capabilities(::LogTargetDensity) LogDensityProblems.LogDensityOrder{1}() end +# suggested extra loss function +function L2loss2(Tar::LogTargetDensity, θ) + f = Tar.prob.f + + # parameter estimation chosen or not + if Tar.extraparams > 0 + dataset, deri_sol = Tar.dataset + # deri_sol = deri_sol' + autodiff = Tar.autodiff + + # # Timepoints to enforce Physics + # dataset = Array(reduce(hcat, dataset)') + # t = dataset[end, :] + # û = dataset[1:(end - 1), :] + + # ode_params = Tar.extraparams == 1 ? + # θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : + # θ[((length(θ) - Tar.extraparams) + 1):length(θ)] + + # if length(û[:, 1]) == 1 + # physsol = [f(û[:, i][1], + # ode_params, + # t[i]) + # for i in 1:length(û[1, :])] + # else + # physsol = [f(û[:, i], + # ode_params, + # t[i]) + # for i in 1:length(û[1, :])] + # end + # #form of NN output matrix output dim x n + # deri_physsol = reduce(hcat, physsol) + + # > for perfect deriv(basically gradient matching in case of an ODEFunction) + # in case of PDE or general ODE we would want to reduce residue of f(du,u,p,t) + # if length(û[:, 1]) == 1 + # deri_sol = [f(û[:, i][1], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[1, :])] + # else + # deri_sol = [f(û[:, i], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[1, :])] + # end + # deri_sol = reduce(hcat, deri_sol) + # deri_sol = reduce(hcat, derivatives) + + # Timepoints to enforce Physics + t = dataset[end] + u1 = dataset[2] + û = dataset[1] + # Tar(t, θ[1:(length(θ) - Tar.extraparams)])' + # + + nnsol = NNodederi(Tar, t, θ[1:(length(θ) - Tar.extraparams)], autodiff) + + ode_params = Tar.extraparams == 1 ? + θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : + θ[((length(θ) - Tar.extraparams) + 1):length(θ)] + + if length(Tar.prob.u0) == 1 + physsol = [f(û[i], + ode_params, + t[i]) + for i in 1:length(û[:, 1])] + else + physsol = [f([û[i], u1[i]], + ode_params, + t[i]) + for i in 1:length(û[:, 1])] + end + #form of NN output matrix output dim x n + deri_physsol = reduce(hcat, physsol) + + # if length(Tar.prob.u0) == 1 + # nnsol = [f(û[i], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[:, 1])] + # else + # nnsol = [f([û[i], u1[i]], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[:, 1])] + # end + # form of NN output matrix output dim x n + # nnsol = reduce(hcat, nnsol) + + # > Instead of dataset gradients trying NN derivatives with dataset collocation + # # convert to matrix as nnsol + + physlogprob = 0 + for i in 1:length(Tar.prob.u0) + # can add phystd[i] for u[i] + physlogprob += logpdf(MvNormal(deri_physsol[i, :], + LinearAlgebra.Diagonal(map(abs2, + (Tar.l2std[i] * 4.0) .* + ones(length(nnsol[i, :]))))), + nnsol[i, :]) + end + return physlogprob + else + return 0 + end +end + +# PDE(DU,U,P,T)=0 + +# Derivated via Central Diff +# function calculate_derivatives2(dataset) +# x̂, time = dataset +# num_points = length(x̂) +# # Initialize an array to store the derivative values. +# derivatives = similar(x̂) + +# for i in 2:(num_points - 1) +# # Calculate the first-order derivative using central differences. +# Δt_forward = time[i + 1] - time[i] +# Δt_backward = time[i] - time[i - 1] + +# derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) + +# derivatives[i] = derivative +# end + +# # Derivatives at the endpoints can be calculated using forward or backward differences. +# derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) +# derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) +# return derivatives +# end + +function calderivatives(prob, dataset) + chainflux = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), + Flux.Dense(8, 2)) |> Flux.f64 + # chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 + function loss(x, y) + # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1]) + + # Flux.mse.(prob.u0[2] .+ (prob.tspan[2] .- x)' .* chainflux(x)[2, :], y[2])) + # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1])) + sum(Flux.mse.(chainflux(x), y)) + end + optimizer = Flux.Optimise.ADAM(0.01) + epochs = 3000 + for epoch in 1:epochs + Flux.train!(loss, + Flux.params(chainflux), + [(dataset[end]', dataset[1:(end - 1)])], + optimizer) + end + + # A1 = (prob.u0' .+ + # (prob.tspan[2] .- (dataset[end]' .+ sqrt(eps(eltype(Float64)))))' .* + # chainflux(dataset[end]' .+ sqrt(eps(eltype(Float64))))') + + # A2 = (prob.u0' .+ + # (prob.tspan[2] .- (dataset[end]'))' .* + # chainflux(dataset[end]')') + + A1 = chainflux(dataset[end]' .+ sqrt(eps(eltype(dataset[end][1])))) + A2 = chainflux(dataset[end]') + + gradients = (A2 .- A1) ./ sqrt(eps(eltype(dataset[end][1]))) + + return gradients +end + +function calculate_derivatives(dataset) + + # u = dataset[1] + # u1 = dataset[2] + # t = dataset[end] + # # control points + # n = Int(floor(length(t) / 10)) + # # spline for datasetvalues(solution) + # # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) + # interp = CubicSpline(u, t) + # interp1 = CubicSpline(u1, t) + # # derrivatives interpolation + # dx = t[2] - t[1] + # time = collect(t[1]:dx:t[end]) + # smoothu = [interp(i) for i in time] + # smoothu1 = [interp1(i) for i in time] + # # derivative of the spline (must match function derivative) + # û = tvdiff(smoothu, 20, 0.5, dx = dx, ε = 1) + # û1 = tvdiff(smoothu1, 20, 0.5, dx = dx, ε = 1) + # # tvdiff(smoothu, 100, 0.035, dx = dx, ε = 1) + # # FDM + # # û1 = diff(u) / dx + # # dataset[1] and smoothu are almost equal(rounding errors) + # return [û, û1] + +end + """ L2 loss loglikelihood(needed for ODE parameter estimation) """ function L2LossData(Tar::LogTargetDensity, θ) + dataset = Tar.dataset # check if dataset is provided - if Tar.dataset isa Vector{Nothing} || Tar.extraparams == 0 + if dataset isa Vector{Nothing} || Tar.extraparams == 0 return 0 else # matrix(each row corresponds to vector u's rows) - nn = Tar(Tar.dataset[end], θ[1:(length(θ) - Tar.extraparams)]) + nn = Tar(dataset[end], θ[1:(length(θ) - Tar.extraparams)]) L2logprob = 0 for i in 1:length(Tar.prob.u0) # for u[i] ith vector must be added to dataset,nn[1,:] is the dx in lotka_volterra L2logprob += logpdf(MvNormal(nn[i, :], LinearAlgebra.Diagonal(map(abs2, - Tar.l2std[i] .* - ones(length(Tar.dataset[i]))))), - Tar.dataset[i]) + (Tar.l2std[i] * 0.5) .* + ones(length(dataset[i]))))), + dataset[i]) end return L2logprob end @@ -174,6 +371,7 @@ function getlogpdf(strategy::QuadratureTraining, Tar::LogTargetDensity, f, innerdiff(Tar, f, autodiff, [t], θ, ode_params) end intprob = IntegralProblem(integrand, tspan[1], tspan[2], θ; nout = length(Tar.prob.u0)) + # add dataset logpdf? sol = solve(intprob, QuadGKJL(); abstol = strategy.abstol, reltol = strategy.reltol) sum(sol.u) end @@ -545,6 +743,11 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; metric = Metric(nparameters) hamiltonian = Hamiltonian(metric, ℓπ, ForwardDiff) + println("physics Logpdf is : ", physloglikelihood(ℓπ, initial_θ)) + println("prior Logpdf is : ", priorweights(ℓπ, initial_θ)) + println("L2lossData Logpdf is : ", L2LossData(ℓπ, initial_θ)) + println("L2loss2 Logpdf is : ", L2loss2(ℓπ, initial_θ)) + # parallel sampling option if nchains != 1 # Cache to store the chains diff --git a/test/BPINN_Tests.jl b/test/BPINN_Tests.jl index b04483015b..fa2f04073e 100644 --- a/test/BPINN_Tests.jl +++ b/test/BPINN_Tests.jl @@ -109,9 +109,9 @@ u = sol1.u time = sol1.t # BPINN AND TRAINING DATASET CREATION(dataset must be defined only inside problem timespan!) -ta = range(tspan[1], tspan[2], length = 100) +ta = range(tspan[1], tspan[2], length = 25) u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +x̂ = collect(Float64, Array(u) .+ (0.2 .* Array(u) .* randn(size(u)))) time = vec(collect(Float64, ta)) dataset = [x̂, time] physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] @@ -123,6 +123,10 @@ x̂1 = collect(Float64, Array(u1) + 0.2 * randn(size(u1))) time1 = vec(collect(Float64, ta0)) physsol1_1 = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] +using Plots, StatsPlots +# plot(dataset[2], calderivatives(dataset)') +yu = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) +plot(yu, [linear_analytic(u0, p, t) for t in yu]) chainflux1 = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 chainlux1 = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) init1, re1 = destructure(chainflux1) @@ -151,13 +155,56 @@ fh_mcmc_chain2, fhsamples2, fhstats2 = ahmc_bayesian_pinn_ode(prob, chainlux1, n_leapfrog = 30) alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 2500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 10.0), + l2std = [0.005], phystd = [0.01], + param = [Normal(11, 6)], Metric = DiagEuclideanMetric, n_leapfrog = 30) - +# original paper (pure data 0 1) +sol1flux = solve(prob, alg) +sol1flux.estimated_ode_params +# pure data method 1 1 sol2flux = solve(prob, alg) +sol2flux.estimated_ode_params +# pure data method 1 0 +sol3flux = solve(prob, alg) +sol3flux.estimated_ode_params +# deri collocation +sol4flux = solve(prob, alg) +sol4flux.estimated_ode_params +# collocation +sol5flux = solve(prob, alg) +sol5flux.estimated_ode_params +# collocation + L2Data loss(at 9,0.5 1,2 gives same) +sol6flux = solve(prob, alg) +sol6flux.estimated_ode_params +# 2500 iters +sol7flux = solve(prob, alg) +sol7flux.estimated_ode_params + +plotly() +plot!(yu, sol1flux.ensemblesol[1]) +plot!(yu, sol2flux.ensemblesol[1]) +plot!(yu, sol3flux.ensemblesol[1]) +plot!(yu, sol4flux.ensemblesol[1]) +plot!(yu, sol5flux.ensemblesol[1]) +plot!(yu, sol6flux.ensemblesol[1]) + +plot!(dataset[2], dataset[1]) + +# plot!(sol4flux.ensemblesol[1]) +# plot!(sol5flux.ensemblesol[1]) + +sol2flux.estimated_ode_params + +sol1flux.estimated_ode_params + +sol3flux.estimated_ode_params + +sol4flux.estimated_ode_params + +sol5flux.estimated_ode_params alg = NeuralPDE.BNNODE(chainlux1, dataset = dataset, draw_samples = 2500, @@ -199,7 +246,7 @@ meanscurve2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean @test mean(abs.(physsol1_1 .- sol2lux.ensemblesol[1])) < 8e-2 # ESTIMATED ODE PARAMETERS (NN1 AND NN2) -@test abs(p - sol2flux.estimated_ode_params[1]) < abs(0.15 * p) +@test abs(p - sol1flux.estimated_ode_params[1]) < abs(0.15 * p) @test abs(p - sol2lux.estimated_ode_params[1]) < abs(0.15 * p) ## PROBLEM-2 @@ -230,6 +277,37 @@ chainlux12 = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), Lux.Dense(6 init1, re1 = destructure(chainflux12) θinit, st = Lux.setup(Random.default_rng(), chainlux12) +using Flux +using Random + +function derivatives(chainflux, dataset) + loss(x, y) = Flux.mse(chainflux(x), y) + optimizer = Flux.Optimise.ADAM(0.01) + epochs = 2500 + for epoch in 1:epochs + Flux.train!(loss, Flux.params(chainflux), [(dataset[2]', dataset[1]')], optimizer) + end + getgradient(chainflux, dataset) +end + +function getgradient(chainflux, dataset) + return (chainflux(dataset[end]' .+ sqrt(eps(eltype(Float64)))) .- + chainflux(dataset[end]')) ./ + sqrt(eps(eltype(dataset[end][1]))) +end + +ans = derivatives(chainflux12, dataset) + +init3, re = destructure(chainflux12) +init2 == init1 +init3 == init2 +plot!(dataset[end], ans') +plot!(dataset[end], chainflux12(dataset[end]')') + +ars = getgradient(chainflux12, dataset) + +plot!(dataset[end], ars') + fh_mcmc_chainflux12, fhsamplesflux12, fhstatsflux12 = ahmc_bayesian_pinn_ode(prob, chainflux12, draw_samples = 1500, @@ -277,10 +355,10 @@ fh_mcmc_chainlux22, fhsampleslux22, fhstatslux22 = ahmc_bayesian_pinn_ode(prob, ], n_leapfrog = 30) -alg = NeuralPDE.BNNODE(chainflux12, +alg1 = NeuralPDE.BNNODE(chainflux12, dataset = dataset, - draw_samples = 1500, - l2std = [0.03], + draw_samples = 500, + l2std = [0.01], phystd = [ 0.03, ], @@ -290,10 +368,50 @@ alg = NeuralPDE.BNNODE(chainflux12, Normal(-7, 4), ], - n_leapfrog = 30) + n_leapfrog = 30, progress = true) + +# original paper (pure data 0 1) +sol1flux_pestim = solve(prob, alg1) +sol1flux_pestim.estimated_ode_params +# pure data method 1 1 +sol2flux_pestim = solve(prob, alg1) +sol2flux_pestim.estimated_ode_params +# pure data method 1 0 +sol3flux_pestim = solve(prob, alg1) +sol3flux_pestim.estimated_ode_params +# deri collocation +sol4flux_pestim = solve(prob, alg1) +sol4flux_pestim.estimated_ode_params +# collocation +sol5flux_pestim = solve(prob, alg1) +sol5flux_pestim.estimated_ode_params +# collocation + L2Data loss(at 9,0.5 1,2 gives same) +sol6flux_pestim = solve(prob, alg1) +sol6flux_pestim.estimated_ode_params + +using Plots, StatsPlots +ars = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) +plot(time, u) +plot!(ars, sol1flux_pestim.ensemblesol[1]) +plot!(ars, sol2flux_pestim.ensemblesol[1]) +plot!(ars, sol3flux_pestim.ensemblesol[1]) +plot!(ars, sol4flux_pestim.ensemblesol[1]) +plot!(ars, sol5flux_pestim.ensemblesol[1]) +plot!(ars, sol6flux_pestim.ensemblesol[1]) + +sol3flux_pestim.estimated_ode_params + +sol4flux_pestim.estimated_ode_params + +sol5flux_pestim.estimated_ode_params -sol3flux_pestim = solve(prob, alg) +sol6flux_pestim.estimated_ode_params +ars = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) + +init, re1 = destructure(chainflux12) +init +init1 alg = NeuralPDE.BNNODE(chainlux12, dataset = dataset, draw_samples = 1500, @@ -363,4 +481,3874 @@ param1 = sol3flux_pestim.estimated_ode_params[1] @test mean(abs.(physsol2 .- sol3lux_pestim.ensemblesol[1])) < 0.15 # estimated parameters(lux chain) param1 = sol3lux_pestim.estimated_ode_params[1] -@test abs(param1 - p) < abs(0.45 * p) \ No newline at end of file +@test abs(param1 - p) < abs(0.45 * p) + +using Plots, StatsPlots +using NoiseRobustDifferentiation, Weave, DataInterpolations + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood +# # 25 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, +# draw_samples = 1500, physdt = 1 / 50.0f0, phystd = [0.01], +# l2std = [0.01], +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux1 = solve(prob, alg) +# sol2flux1.estimated_ode_params[1] #6.41722 Particles{Float64, 1}, 6.02404 Particles{Float64, 1} +# sol2flux2 = solve(prob, alg) +# sol2flux2.estimated_ode_params[1] #6.42782 Particles{Float64, 1}, 6.07509 Particles{Float64, 1} +# sol2flux3 = solve(prob, alg) +# sol2flux3.estimated_ode_params[1] #6.42782 Particles{Float64, 1}, 6.00825 Particles{Float64, 1} + +# # 50 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux11 = solve(prob, alg) +# sol2flux11.estimated_ode_params[1] #5.71268 Particles{Float64, 1}, 6.07242 Particles{Float64, 1} +# sol2flux22 = solve(prob, alg) +# sol2flux22.estimated_ode_params[1] #5.74599 Particles{Float64, 1}, 6.04837 Particles{Float64, 1} +# sol2flux33 = solve(prob, alg) +# sol2flux33.estimated_ode_params[1] #5.74599 Particles{Float64, 1}, 6.02838 Particles{Float64, 1} + +# # 100 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux111 = solve(prob, alg) +# sol2flux111.estimated_ode_params[1] #6.59097 Particles{Float64, 1}, 5.89384 Particles{Float64, 1} +# sol2flux222 = solve(prob, alg) +# sol2flux222.estimated_ode_params[1] #6.62813 Particles{Float64, 1}, 5.88216 Particles{Float64, 1} +# sol2flux333 = solve(prob, alg) +# sol2flux333.estimated_ode_params[1] #6.62813 Particles{Float64, 1}, 5.85327 Particles{Float64, 1} + +# # ---------------------------------------------------------- +# # physdt=1/20, full likelihood cdm +# # 25 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux1_cdm = solve(prob, alg) +# sol2flux1_cdm.estimated_ode_params[1]# 6.50506 Particles{Float64, 1} ,6.38963 Particles{Float64, 1} +# sol2flux2_cdm = solve(prob, alg) +# sol2flux2_cdm.estimated_ode_params[1] #6.50032 Particles{Float64, 1} ,6.39817 Particles{Float64, 1} +# sol2flux3_cdm = solve(prob, alg) +# sol2flux3_cdm.estimated_ode_params[1] #6.50032 Particles{Float64, 1} ,6.36296 Particles{Float64, 1} + +# # 50 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux11_cdm = solve(prob, alg) +# sol2flux11_cdm.estimated_ode_params[1] #6.52951 Particles{Float64, 1},5.15621 Particles{Float64, 1} +# sol2flux22_cdm = solve(prob, alg) +# sol2flux22_cdm.estimated_ode_params[1] #6.54988 Particles{Float64, 1},5.16363 Particles{Float64, 1} +# sol2flux33_cdm = solve(prob, alg) +# sol2flux33_cdm.estimated_ode_params[1] #6.54988 Particles{Float64, 1},5.15591 Particles{Float64, 1} + +# # 100 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux111_cdm = solve(prob, alg) +# sol2flux111_cdm.estimated_ode_params[1] #6.74338 Particles{Float64, 1}, 9.72422 Particles{Float64, 1} +# sol2flux222_cdm = solve(prob, alg) +# sol2flux222_cdm.estimated_ode_params[1] #6.72642 Particles{Float64, 1}, 9.71991 Particles{Float64, 1} +# sol2flux333_cdm = solve(prob, alg) +# sol2flux333_cdm.estimated_ode_params[1] #6.72642 Particles{Float64, 1}, 9.75045 Particles{Float64, 1} + +# -------------------------------------------------------------------------------------- +# NEW SERIES OF TESTS (IN ORDER OF EXECUTION) +# ------------------------------------------------------------------------------------- +# original paper implementaion +# 25 points +ta = range(tspan[1], tspan[2], length = 25) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, u .+ 0.05 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset1 = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] +# scatter!(time, u) +# dataset +# scatter!(dataset1[2], dataset1[1]) +# plot(time, physsol1) + +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_normal = solve(prob, alg) +sol2flux1_normal.estimated_ode_params[1] #7.70593 Particles{Float64, 1}, 6.36096 Particles{Float64, 1} | 6.45865 Particles{Float64, 1} +sol2flux2_normal = solve(prob, alg) +sol2flux2_normal.estimated_ode_params[1] #6.66347 Particles{Float64, 1}, 6.36974 Particles{Float64, 1} | 6.45865 Particles{Float64, 1} +sol2flux3_normal = solve(prob, alg) +sol2flux3_normal.estimated_ode_params[1] #6.84827 Particles{Float64, 1}, 6.29555 Particles{Float64, 1} | 6.39947 Particles{Float64, 1} + +# 50 points +ta = range(tspan[1], tspan[2], length = 50) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, Array(u) + 0.05 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset2 = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_normal = solve(prob, alg) +sol2flux11_normal.estimated_ode_params[1] #7.83577 Particles{Float64, 1},6.24652 Particles{Float64, 1} | 6.34495 Particles{Float64, 1} +sol2flux22_normal = solve(prob, alg) +sol2flux22_normal.estimated_ode_params[1] #6.49477 Particles{Float64, 1},6.2118 Particles{Float64, 1} | 6.32476 Particles{Float64, 1} +sol2flux33_normal = solve(prob, alg) +sol2flux33_normal.estimated_ode_params[1] #6.47421 Particles{Float64, 1},6.33687 Particles{Float64, 1} | 6.2448 Particles{Float64, 1} + +# 100 points +ta = range(tspan[1], tspan[2], length = 100) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, Array(u) + 0.05 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset3 = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_normal = solve(prob, alg) +sol2flux111_normal.estimated_ode_params[1] #5.96604 Particles{Float64, 1},5.99588 Particles{Float64, 1} | 6.19805 Particles{Float64, 1} +sol2flux222_normal = solve(prob, alg) +sol2flux222_normal.estimated_ode_params[1] #6.05432 Particles{Float64, 1},6.0768 Particles{Float64, 1} | 6.22948 Particles{Float64, 1} +sol2flux333_normal = solve(prob, alg) +sol2flux333_normal.estimated_ode_params[1] #6.08856 Particles{Float64, 1},5.94819 Particles{Float64, 1} | 6.2551 Particles{Float64, 1} + +# LOTKA VOLTERRA CASE +function lotka_volterra(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end + +# initial-value problem. +u01 = [1.0, 1.0] +p1 = [1.5, 1.0, 3.0, 1.0] +tspan1 = (0.0, 6.0) +prob1 = ODEProblem(lotka_volterra, u01, tspan1, p1) + +# chainlux = Lux.Chain(Lux.Dense(1, 7, Lux.tanh), Lux.Dense(7, 7, Lux.tanh), Lux.Dense(7, 2)) +chainflux1 = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), Flux.Dense(8, 2)) + +#testing timepoints must match keyword arg `saveat`` timepoints of solve() call +t1 = collect(Float64, prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]) + +# -------------------------------------------------------------------------- +# original paper implementaion lotka volterra +# 31 points +solution1 = solve(prob1, Tsit5(); saveat = 0.1) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] .+ 0.3 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] .+ 0.3 .* u1[2, :] .* randn(length(u1[2, :])) +dataset2_1 = [x1, y1, time1] +plot(dataset2_1[end], dataset2_1[1]) +plot!(dataset2_1[end], dataset2_1[2]) +plot!(time1, u1[1, :]) +plot!(time1, u1[2, :]) + +alg1 = NeuralPDE.BNNODE(chainflux1, + dataset = dataset2_1, + draw_samples = 1000, + physdt = 1 / 20.0, + l2std = [ + 0.2, + 0.2, + ], + phystd = [ + 0.5, + 0.5, + ], + priorsNNw = (0.0, + 10.0), + param = [ + Normal(4, + 3), + Normal(-2, + 4), + Normal(0, + 5), + Normal(2.5, + 2)], + n_leapfrog = 30, progress = true) + +# original paper (pure data 0 1) +sol1flux1_lotka = solve(prob1, alg1) +sol1flux1_lotka.estimated_ode_params +# pure data method 1 1 +sol2flux1_lotka = solve(prob1, alg1) +sol2flux1_lotka.estimated_ode_params +# pure data method 1 0 +sol3flux1_lotka = solve(prob1, alg1) +sol3flux1_lotka.estimated_ode_params +# deri collocation +sol4flux1_lotka = solve(prob1, alg1) +sol4flux1_lotka.estimated_ode_params +# collocation +sol5flux1_lotka = solve(prob1, alg1) +sol5flux1_lotka.estimated_ode_params +# collocation + L2Data loss(at 9,0.5 1,2 gives same) +sol6flux1_lotka = solve(prob1, alg1) +sol6flux1_lotka.estimated_ode_params + +sol7flux1_lotka = solve(prob1, alg1) +sol7flux1_lotka.estimated_ode_params + +using Plots, StatsPlots +plot(dataset2_1[3], u1[1, :]) +plot!(dataset2_1[3], u1[2, :]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol5flux1_normal.ensemblesol[2]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), + sol1flux1_normal.ensemblesol[1], + legend = :outerbottomleft) +sol1flux2_normal = solve(prob1, alg1) +sol1flux2_normal.estimated_ode_params #| +sol1flux3_normal = solve(prob1, alg1) +sol1flux3_normal.estimated_ode_params #| +sol1flux4_normal = solve(prob1, alg1) +sol1flux4_normal.estimated_ode_params + +plotly() +plot!(title = "yuh") +plot!(dataset2_1[3], dataset2_1[1]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux1_normal.ensemblesol[1]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux2_normal.ensemblesol[1]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux3_normal.ensemblesol[2]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux4_normal.ensemblesol[1]) +plot(time1, u1[1, :]) +plot!(time1, u1[2, :]) + +ars = chainflux1(dataset2_1[end]') +plot(ars[1, :]) +plot!(ars[2, :]) + +function calculate_derivatives(dataset) + u = dataset[1] + u1 = dataset[2] + t = dataset[end] + # control points + n = Int(floor(length(t) / 10)) + # spline for datasetvalues(solution) + # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) + interp = CubicSpline(u, t) + interp1 = CubicSpline(u1, t) + # derrivatives interpolation + dx = t[2] - t[1] + time = collect(t[1]:dx:t[end]) + smoothu = [interp(i) for i in time] + smoothu1 = [interp1(i) for i in time] + # derivative of the spline (must match function derivative) + û = tvdiff(smoothu, 20, 0.5, dx = dx, ε = 1) + û1 = tvdiff(smoothu1, 20, 0.5, dx = dx, ε = 1) + # tvdiff(smoothu, 100, 0.035, dx = dx, ε = 1) + # FDM + # û1 = diff(u) / dx + # dataset[1] and smoothu are almost equal(rounding errors) + return û, û1 + # return 1 +end + +ar = calculate_derivatives(dataset2_1) +plot(ar[1]) +plot!(ar[2]) + +# 61 points +solution1 = solve(prob1, Tsit5(); saveat = 0.1) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +dataset2_2 = [x1, y1, time1] + +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_2, + draw_samples = 1000, + l2std = [ + 0.1, + 0.1, + ], + phystd = [ + 0.1, + 0.1, + ], + priorsNNw = (0.0, + 5.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux11_normal = solve(prob1, alg1) +sol1flux11_normal.estimated_ode_params #| +sol1flux22_normal = solve(prob1, alg1) +sol1flux22_normal.estimated_ode_params #| +sol1flux33_normal = solve(prob1, alg1) +sol1flux33_normal.estimated_ode_params #| + +# 121 points +solution1 = solve(prob1, Tsit5(); saveat = 0.05) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +dataset2_3 = [x1, y1, time1] + +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_3, + draw_samples = 1000, + l2std = [ + 0.1, + 0.1, + ], + phystd = [ + 0.1, + 0.1, + ], + priorsNNw = (0.0, + 5.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux111_normal = solve(prob1, alg1) +sol1flux111_normal.estimated_ode_params #| +sol1flux222_normal = solve(prob1, alg1) +sol1flux222_normal.estimated_ode_params #| +sol1flux333_normal = solve(prob1, alg1) +sol1flux333_normal.estimated_ode_params #| + +# -------------------------------------------------------------------- + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# Sampling 100%|███████████████████████████████| Time: 0:02:30 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# Sampling 100%|███████████████████████████████| Time: 0:01:54 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# Sampling 100%|███████████████████████████████| Time: 0:01:59 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# Sampling 100%|███████████████████████████████| Time: 0:02:44 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# Sampling 100%|███████████████████████████████| Time: 0:02:41 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# Sampling 100%|███████████████████████████████| Time: 0:02:41 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# Sampling 100%|███████████████████████████████| Time: 0:03:52 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# Sampling 100%|███████████████████████████████| Time: 0:03:49 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# Sampling 100%|███████████████████████████████| Time: 0:03:50 + +# # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +# physics Logpdf is : -6.659143464386241e7 +# prior Logpdf is : -150.30074579848434 +# L2lossData Logpdf is : -6.03075717462954e6 +# Sampling 100%|███████████████████████████████| Time: 0:04:54 + +# physics Logpdf is : -8.70012053004202e8 +# prior Logpdf is : -150.3750892952511 +# L2lossData Logpdf is : -6.967914805207133e6 +# Sampling 100%|███████████████████████████████| Time: 0:05:09 + +# physics Logpdf is : -5.417241281343099e7 +# prior Logpdf is : -150.52079555737976 +# L2lossData Logpdf is : -4.195953436792884e6 +# Sampling 100%|███████████████████████████████| Time: 0:05:01 + +# physics Logpdf is : -4.579552981943833e8 +# prior Logpdf is : -150.30491731974283 +# L2lossData Logpdf is : -8.595475827260146e6 +# Sampling 100%|███████████████████████████████| Time: 0:06:08 + +# physics Logpdf is : -1.989281834955769e7 +# prior Logpdf is : -150.16009042727543 +# L2lossData Logpdf is : -1.121270659669029e7 +# Sampling 100%|███████████████████████████████| Time: 0:05:38 + +# physics Logpdf is : -8.683829147264534e8 +# prior Logpdf is : -150.37824872259102 +# L2lossData Logpdf is : -1.0887662888035845e7 +# Sampling 100%|███████████████████████████████| Time: 0:05:50 + +# physics Logpdf is : -3.1944760610332566e8 +# prior Logpdf is : -150.33610348737565 +# L2lossData Logpdf is : -1.215458786744478e7 +# Sampling 100%|███████████████████████████████| Time: 0:10:50 + +# physics Logpdf is : -3.2884572300341567e6 +# prior Logpdf is : -150.21002268156343 +# L2lossData Logpdf is : -1.102536731511176e7 +# Sampling 100%|███████████████████████████████| Time: 0:09:53 + +# physics Logpdf is : -5.31293521002414e8 +# prior Logpdf is : -150.20948536040126 +# L2lossData Logpdf is : -1.818717239584132e7 +# Sampling 100%|███████████████████████████████| Time: 0:08:53 + +# ---------------------------------------------------------- +# Full likelihood no l2 only new L22(NN gradients) +# 25 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_new = solve(prob, alg) +sol2flux1_new.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 6.21662 Particles{Float64, 1} +sol2flux2_new = solve(prob, alg) +sol2flux2_new.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 7.14238 Particles{Float64, 1} +sol2flux3_new = solve(prob, alg) +sol2flux3_new.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 6.79159 Particles{Float64, 1} + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_new = solve(prob, alg) +sol2flux11_new.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 5.33467 Particles{Float64, 1} +sol2flux22_new = solve(prob, alg) +sol2flux22_new.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.52419 Particles{Float64, 1} +sol2flux33_new = solve(prob, alg) +sol2flux33_new.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 5.36921 Particles{Float64, 1} + +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_new = solve(prob, alg) +sol2flux111_new.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.45333 Particles{Float64, 1} +sol2flux222_new = solve(prob, alg) +sol2flux222_new.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 4.64417 Particles{Float64, 1} +sol2flux333_new = solve(prob, alg) +sol2flux333_new.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | 5.88037 Particles{Float64, 1} +# --------------------------------------------------------------------------- + +# ---------------------------------------------------------- +# Full likelihood l2 + new L22(NN gradients) +# 25 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_new_all = solve(prob, alg) +sol2flux1_new_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 6.4358 Particles{Float64, 1} +sol2flux2_new_all = solve(prob, alg) +sol2flux2_new_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 6.52449 Particles{Float64, 1} +sol2flux3_new_all = solve(prob, alg) +sol2flux3_new_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 6.34188 Particles{Float64, 1} + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_new_all = solve(prob, alg) +sol2flux11_new_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 6.37889 Particles{Float64, 1} +sol2flux22_new_all = solve(prob, alg) +sol2flux22_new_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.34747 Particles{Float64, 1} +sol2flux33_new_all = solve(prob, alg) +sol2flux33_new_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 6.39699 Particles{Float64, 1} + +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_new_all = solve(prob, alg) +sol2flux111_new_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.24327 Particles{Float64, 1} +sol2flux222_new_all = solve(prob, alg) +sol2flux222_new_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 6.23928 Particles{Float64, 1} +sol2flux333_new_all = solve(prob, alg) +sol2flux333_new_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | 6.2145 Particles{Float64, 1} + +# --------------------------------------------------------------------------- +# Full likelihood l2 + new L22(dataset gradients) lotka volterra +# 36 points +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_1, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux1_new_all = solve(prob1, alg1) +sol1flux1_new_all.estimated_ode_params[1] #| +sol1flux2_new_all = solve(prob1, alg1) +sol1flux2_new_all.estimated_ode_params[1] #| +sol1flux3_new_all = solve(prob1, alg1) +sol1flux3_new_all.estimated_ode_params[1] #| + +# 61 points +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_2, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux11_new_all = solve(prob1, alg1) +sol1flux11_new_all.estimated_ode_params[1] #| +sol1flux22_new_all = solve(prob1, alg1) +sol1flux22_new_all.estimated_ode_params[1] #| +sol1flux33_new_all = solve(prob1, alg1) +sol1flux33_new_all.estimated_ode_params[1] #| + +# 121 points +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_3, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux111_new_all = solve(prob1, alg1) +sol1flux111_new_all.estimated_ode_params[1] #| +sol1flux222_new_all = solve(prob1, alg1) +sol1flux222_new_all.estimated_ode_params[1] #| +sol1flux333_new_all = solve(prob1, alg1) +sol1flux333_new_all.estimated_ode_params[1] #| +# -------------------------------------------------------------------- + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# L2loss2 Logpdf is : -757.9047847584478 +# Sampling 100%|███████████████████████████████| Time: 0:02:32 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# L2loss2 Logpdf is : -757.9047847584478 +# Sampling 100%|███████████████████████████████| Time: 0:02:19 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# L2loss2 Logpdf is : -757.9047847584478 +# Sampling 100%|███████████████████████████████| Time: 0:02:31 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# L2loss2 Logpdf is : -1517.3653615845183 +# Sampling 100%|███████████████████████████████| Time: 0:03:45 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# L2loss2 Logpdf is : -1517.3653615845183 +# Sampling 100%|███████████████████████████████| Time: 0:03:20 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# L2loss2 Logpdf is : -1517.3653615845183 +# Sampling 100%|███████████████████████████████| Time: 0:03:20 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# L2loss2 Logpdf is : -3037.8868319811254 +# Sampling 100%|███████████████████████████████| Time: 0:04:57 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# L2loss2 Logpdf is : -3037.8868319811254 +# Sampling 100%|███████████████████████████████| Time: 0:05:26 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# L2loss2 Logpdf is : -3037.8868319811254 +# Sampling 100%|███████████████████████████████| Time: 0:05:01 + +# ---------------------------------------------------------- +# Full likelihood l2 + new L22(dataset gradients) +# 25 points +# 1*,2*, +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_newdata_all = solve(prob, alg) +sol2flux1_newdata_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 5.73072 Particles{Float64, 1} +sol2flux2_newdata_all = solve(prob, alg) +sol2flux2_newdata_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 5.71597 Particles{Float64, 1} +sol2flux3_newdata_all = solve(prob, alg) +sol2flux3_newdata_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 5.7313 Particles{Float64, 1} + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_newdata_all = solve(prob, alg) +sol2flux11_newdata_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 6.07153 Particles{Float64, 1} +sol2flux22_newdata_all = solve(prob, alg) +sol2flux22_newdata_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.06623 Particles{Float64, 1} +sol2flux33_newdata_all = solve(prob, alg) +sol2flux33_newdata_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 6.12748 Particles{Float64, 1} + +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_newdata_all = solve(prob, alg) +sol2flux111_newdata_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.26222 Particles{Float64, 1} +sol2flux222_newdata_all = solve(prob, alg) +sol2flux222_newdata_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 5.86494 Particles{Float64, 1} +sol2flux333_newdata_all = solve(prob, alg) +sol2flux333_newdata_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | + +# --------------------------------------------------------------------------- + +# LOTKA VOLTERRA CASE +using Plots, StatsPlots +function lotka_volterra(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end + +# initial-value problem. +u01 = [1.0, 1.0] +p1 = [1.5, 1.0, 3.0, 1.0] +tspan1 = (0.0, 6.0) +prob1 = ODEProblem(lotka_volterra, u01, tspan1, p1) + +chainlux = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), Lux.Dense(6, 2)) + +#testing timepoints must match keyword arg `saveat`` timepoints of solve() call +t1 = collect(Float64, prob.tspan[1]:(1 / 50.0):prob.tspan[2]) + +# -------------------------------------------------------------------------- +# original paper implementaion +# 25 points +solution1 = solve(prob1, Tsit5(); saveat = 0.2) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +dataset2_1 = [x1, y1, time1] + +plot(time1, u1[1, :]) +plot!(time1, u1[2, :]) +scatter!(dataset2_1[3], dataset2_1[1]) +scatter!(dataset2_1[3], dataset2_1[2]) + +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_1, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux1_normal = solve(prob1, alg1) +sol1flux1_normal.estimated_ode_params[1] #| +sol1flux2_normal = solve(prob1, alg1) +sol1flux2_normal.estimated_ode_params[1] #| +sol1flux3_normal = solve(prob1, alg1) +sol1flux3_normal.estimated_ode_params[1] #| + +# 50 points +solution1 = solve(prob1, Tsit5(); saveat = 0.05) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +dataset2_2 = [x1, y1, time1] + +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_2, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux11_normal = solve(prob1, alg1) +sol1flux11_normal.estimated_ode_params[1] #| +sol1flux22_normal = solve(prob1, alg1) +sol1flux22_normal.estimated_ode_params[1] #| +sol1flux33_normal = solve(prob1, alg1) +sol1flux33_normal.estimated_ode_params[1] #| + +# 100 points +solution = solve(prob1, Tsit5(); saveat = 0.05) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +dataset2_3 = [x1, y1, time1] + +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_3, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux111_normal = solve(prob1, alg1) +sol1flux111_normal.estimated_ode_params[1] #| +sol1flux222_normal = solve(prob1, alg1) +sol1flux222_normal.estimated_ode_params[1] #| +sol1flux333_normal = solve(prob1, alg1) +sol1flux333_normal.estimated_ode_params[1] #| + +# -------------------------------------------------------------------- + +# ---------------------------------------------------------- +# Full likelihood no l2 only new L22(NN gradients) +# 25 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_new = solve(prob, alg) +sol2flux1_new.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | +sol2flux2_new = solve(prob, alg) +sol2flux2_new.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | +sol2flux3_new = solve(prob, alg) +sol2flux3_new.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_new = solve(prob, alg) +sol2flux11_new.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | +sol2flux22_new = solve(prob, alg) +sol2flux22_new.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | +sol2flux33_new = solve(prob, alg) +sol2flux33_new.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | + +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_new = solve(prob, alg) +sol2flux111_new.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | +sol2flux222_new = solve(prob, alg) +sol2flux222_new.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | +sol2flux333_new = solve(prob, alg) +sol2flux333_new.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | +# --------------------------------------------------------------------------- + +# ---------------------------------------------------------- +# Full likelihood l2 + new L22(NN gradients) +# 25 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_new_all = solve(prob, alg) +sol2flux1_new_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | +sol2flux2_new_all = solve(prob, alg) +sol2flux2_new_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | +sol2flux3_new_all = solve(prob, alg) +sol2flux3_new_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_new_all = solve(prob, alg) +sol2flux11_new_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | +sol2flux22_new_all = solve(prob, alg) +sol2flux22_new_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | +sol2flux33_new_all = solve(prob, alg) +sol2flux33_new_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_new_all = solve(prob, alg) +sol2flux111_new_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | +sol2flux222_new_all = solve(prob, alg) +sol2flux222_new_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | +sol2flux333_new_all = solve(prob, alg) +sol2flux333_new_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | + +# --------------------------------------------------------------------------- + +# ---------------------------------------------------------- +# Full likelihood l2 + new L22(dataset gradients) +# 25 points +# *1,*2 vs *2.5 +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol1flux1_newdata_all = solve(prob, alg) +sol1flux1_newdata_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | +sol1flux2_newdata_all = solve(prob, alg) +sol1flux2_newdata_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | +sol1flux3_newdata_all = solve(prob, alg) +sol1flux3_newdata_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol1flux11_newdata_all = solve(prob, alg) +sol1flux11_newdata_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | +sol1flux22_newdata_all = solve(prob, alg) +sol1flux22_newdata_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | +sol1flux33_newdata_all = solve(prob, alg) +sol1flux33_newdata_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | + +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol1flux111_newdata_all = solve(prob, alg) +sol1flux111_newdata_all.estimated_ode_params[1] #| +sol1flux222_newdata_all = solve(prob, alg) +sol1flux222_newdata_all.estimated_ode_params[1] #| +sol1flux333_newdata_all = solve(prob, alg) +sol1flux333_newdata_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | + +# ------------------------------------------------------------------------------------------------------------------------------ + +# sol2flux111.estimated_ode_params[1] +# # mine *5 +# 7.03386Particles{Float64, 1} +# # normal +# 6.38951Particles{Float64, 1} +# 6.67657Particles{Float64, 1} +# # mine *10 +# 7.53672Particles{Float64, 1} +# # mine *2 +# 6.29005Particles{Float64, 1} +# 6.29844Particles{Float64, 1} + +# # new mine *2 +# 6.39008Particles{Float64, 1} +# 6.22071Particles{Float64, 1} +# 6.15611Particles{Float64, 1} + +# # new mine *2 tvdiff(smoothu, 20, 0.035, dx = dx, ε = 1e-2) +# 6.25549Particles{Float64, 1} +# ---------------------------------------------------------- + +# --------------------------------------------------- + +function calculate_derivatives1(dataset) + x̂, time = dataset + num_points = length(x̂) + # Initialize an array to store the derivative values. + derivatives = similar(x̂) + + for i in 2:(num_points - 1) + # Calculate the first-order derivative using central differences. + Δt_forward = time[i + 1] - time[i] + Δt_backward = time[i] - time[i - 1] + + derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) + + derivatives[i] = derivative + end + + # Derivatives at the endpoints can be calculated using forward or backward differences. + derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) + derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) + return derivatives +end + +function calculate_derivatives2(dataset) + u = dataset[1] + t = dataset[2] + # control points + n = Int(floor(length(t) / 10)) + # spline for datasetvalues(solution) + # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) + interp = CubicSpline(u, t) + # derrivatives interpolation + dx = t[2] - t[1] + time = collect(t[1]:dx:t[end]) + smoothu = [interp(i) for i in time] + # derivative of the spline (must match function derivative) + û = tvdiff(smoothu, 20, 0.03, dx = dx, ε = 1) + # tvdiff(smoothu, 100, 0.1, dx = dx) + # + # + # FDM + û1 = diff(u) / dx + # dataset[1] and smoothu are almost equal(rounding errors) + return û, time, smoothu, û1 +end + +# need to do this for all datasets +c = [linear(prob.u0, p, t) for t in dataset3[2]] #ideal case +b = calculate_derivatives1(dataset2) #central diffs +# a = calculate_derivatives2(dataset) #tvdiff(smoothu, 100, 0.1, dx = dx) +d = calculate_derivatives2(dataset1) #tvdiff(smoothu, 20, 0.035, dx = dx, ε = 1e-2) +d = calculate_derivatives2(dataset2) +d = calculate_derivatives2(dataset3) +mean(abs2.(c .- b)) +mean(abs2.(c .- d[1])) +loss(model, x, y) = mean(abs2.(model(x) .- y)); +scatter!(prob.u0 .+ (prob.tspan[2] .- dataset3[2]) .* chainflux1(dataset3[2]')') +loss(chainflux1, dataset3[2]', dataset3[1]') +# mean(abs2.(c[1:24] .- a[4])) +plot(c, label = "ideal deriv") +plot!(b, label = "Centraldiff deriv") +# plot!(a[1], label = "tvdiff(0.1,def) derivatives") +plot!(d[1], label = "tvdiff(0.035,20) derivatives") +plotly() + +# GridTraining , NoiseRobustDiff dataset[2][2]-dataset[2][1] l2std +# 25 points +ta = range(tspan[1], tspan[2], length = 25) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +time1 = collect(tspan[1]:(1 / 50.0):tspan[2]) +physsol = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] +plot(physsol, label = "solution") + +# plots from 32(deriv) +# for d +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 2000, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux1 = solve(prob, alg) +n2_sol2flux1.estimated_ode_params[1] +# with extra likelihood +# 10.2011Particles{Float64, 1} + +# without extra likelihood +# 6.25791Particles{Float64, 1} +# 6.29539Particles{Float64, 1} + +plot!(n2_sol2flux1.ensemblesol[1], label = "tvdiff(0.035,1) derivpar") +plot(dataset[1]) +plot!(physsol1) +# for a +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 2000, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux2 = solve(prob, alg) +n2_sol2flux2.estimated_ode_params[1] +# with extra likelihood +# 8.73602Particles{Float64, 1} +# without extra likelihood + +plot!(n2_sol2flux2.ensemblesol[1], + label = "tvdiff(0.1,def) derivatives", + legend = :outerbottomleft) + +# for b +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 2000, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux3 = solve(prob, alg) +n2_sol2flux3.estimated_ode_params[1] +plot!(n2_sol2flux3.ensemblesol[1], label = "Centraldiff deriv") + +# for c +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 2000, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux4 = solve(prob, alg) +n2_sol2flux4.estimated_ode_params[1] +plot!(n2_sol2flux4.ensemblesol[1], label = "ideal deriv") + +# 50 points + +ta = range(tspan[1], tspan[2], length = 50) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux11 = solve(prob, alg) +n2_sol2flux11.estimated_ode_params[1] + +# 5.90049Particles{Float64, 1} +# 100 points +ta = range(tspan[1], tspan[2], length = 100) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux111 = solve(prob, alg) +n2_sol2flux111.estimated_ode_params[1] +plot!(n2_sol2flux111.ensemblesol[1]) +8.88555Particles{Float64, 1} + +# 7.15353Particles{Float64, 1} +# 6.21059 Particles{Float64, 1} +# 6.31836Particles{Float64, 1} +0.1 * p +# ---------------------------------------------------------- + +# Gives the linear interpolation value at t=3.5 + +# # Problem 1 with param esimation +# # dataset 0-1 2 percent noise +# p = 6.283185307179586 +# # partial_logdensity +# 6.3549Particles{Float64, 1} +# # full log_density +# 6.34667Particles{Float64, 1} + +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2lux.estimated_ode_params[1] + +# # dataset 0-1 20 percent noise +# # partial log_density +# 6.30244Particles{Float64, 1} +# # full log_density +# 6.24637Particles{Float64, 1} + +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # dataset 0-2 20percent noise +# # partial log_density +# 6.24948Particles{Float64, 1} +# # full log_density +# 6.26095Particles{Float64, 1} + +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# linear_analytic = (u0, p, t) -> u0 + sin(p * t) / (p) +# linear = (u, p, t) -> cos(p * t) +# tspan = (0.0, 2.0) + +# # dataset 0-1 2 percent noise +# p = 6.283185307179586 +# # partial_logdensity +# 6.3549Particles{Float64, 1} +# # full log_density +# 6.34667Particles{Float64, 1} + +# # dataset 0-1 20 percent noise +# # partial log_density +# 6.30244Particles{Float64, 1} +# # full log_density +# 6.24637Particles{Float64, 1} + +# # dataset 0-2 20percent noise +# # partial log_density +# 6.24948Particles{Float64, 1} +# # full log_density +# 6.26095Particles{Float64, 1} + +# # dataset 0-2 20percent noise 50 points(above all are 100 points) +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # partial log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# # i kinda win on 25 points again +# # dataset 0-2 20percent noise 25 points +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # partial log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # i win with 25 points +# # dataset 0-1 20percent noise 25 points +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# # new +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # partial log_density +# sol2flux.estimated_ode_params[1] +# # New +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # (9,2.5)(above are (9,0.5)) +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# # just prev was repeat(just change) +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # partial log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # i lose on 0-1,50 points +# # dataset 0-1 20percent noise 50 points +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# # partial log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # (9,2.5) (above are (9,0.5)) +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # partial log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # ---------------------------------------------------------- +# # Problem 1 with param estimation +# # physdt=1/20, Full likelihood new 0.5*l2std +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n05_sol2flux1 = solve(prob, alg) +# n05_sol2flux1.estimated_ode_params[1] #6.90953 Particles{Float64, 1} +# n05_sol2flux2 = solve(prob, alg) +# n05_sol2flux2.estimated_ode_params[1] #6.82374 Particles{Float64, 1} +# n05_sol2flux3 = solve(prob, alg) +# n05_sol2flux3.estimated_ode_params[1] #6.84465 Particles{Float64, 1} + +# using Plots, StatsPlots +# plot(n05_sol2flux3.ensemblesol[1]) +# plot!(physsol1) +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n05_sol2flux11 = solve(prob, alg) +# n05_sol2flux11.estimated_ode_params[1] #7.0262 Particles{Float64, 1} +# n05_sol2flux22 = solve(prob, alg) +# n05_sol2flux22.estimated_ode_params[1] #5.56438 Particles{Float64, 1} +# n05_sol2flux33 = solve(prob, alg) +# n05_sol2flux33.estimated_ode_params[1] #7.27189 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n05_sol2flux111 = solve(prob, alg) +# n05_sol2flux111.estimated_ode_params[1] #6.90549 Particles{Float64, 1} +# n05_sol2flux222 = solve(prob, alg) +# n05_sol2flux222.estimated_ode_params[1] #5.42436 Particles{Float64, 1} +# n05_sol2flux333 = solve(prob, alg) +# n05_sol2flux333.estimated_ode_params[1] #6.05832 Particles{Float64, 1} + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood new 2*l2std +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2_sol2flux1 = solve(prob, alg) +# n2_sol2flux1.estimated_ode_params[1]#6.9087 Particles{Float64, 1} +# n2_sol2flux2 = solve(prob, alg) +# n2_sol2flux2.estimated_ode_params[1]#6.86507 Particles{Float64, 1} +# n2_sol2flux3 = solve(prob, alg) +# n2_sol2flux3.estimated_ode_params[1]#6.59206 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2_sol2flux11 = solve(prob, alg) +# n2_sol2flux11.estimated_ode_params[1]#7.3715 Particles{Float64, 1} +# n2_sol2flux22 = solve(prob, alg) +# n2_sol2flux22.estimated_ode_params[1]#9.84477 Particles{Float64, 1} +# n2_sol2flux33 = solve(prob, alg) +# n2_sol2flux33.estimated_ode_params[1]#6.87107 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2_sol2flux111 = solve(prob, alg) +# n2_sol2flux111.estimated_ode_params[1]#6.60739 Particles{Float64, 1} +# n2_sol2flux222 = solve(prob, alg) +# n2_sol2flux222.estimated_ode_params[1]#7.05923 Particles{Float64, 1} +# n2_sol2flux333 = solve(prob, alg) +# n2_sol2flux333.estimated_ode_params[1]#6.5017 Particles{Float64, 1} + +# # ---------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood new all 2*l2std +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2all5sol2flux1 = solve(prob, alg) +# n2all5sol2flux1.estimated_ode_params[1]#11.3659 Particles{Float64, 1} +# n2all5sol2flux2 = solve(prob, alg) +# n2all5sol2flux2.estimated_ode_params[1]#6.65634 Particles{Float64, 1} +# n2all5sol2flux3 = solve(prob, alg) +# n2all5sol2flux3.estimated_ode_params[1]#6.61905 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2all5sol2flux11 = solve(prob, alg) +# n2all5sol2flux11.estimated_ode_params[1]#6.27555 Particles{Float64, 1} +# n2all5sol2flux22 = solve(prob, alg) +# n2all5sol2flux22.estimated_ode_params[1]#6.24352 Particles{Float64, 1} +# n2all5sol2flux33 = solve(prob, alg) +# n2all5sol2flux33.estimated_ode_params[1]#6.33723 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2all5sol2flux111 = solve(prob, alg) +# n2all5sol2flux111.estimated_ode_params[1] #5.95535 Particles{Float64, 1} +# n2all5sol2flux222 = solve(prob, alg) +# n2all5sol2flux222.estimated_ode_params[1] #5.98301 Particles{Float64, 1} +# n2all5sol2flux333 = solve(prob, alg) +# n2all5sol2flux333.estimated_ode_params[1] #5.9081 Particles{Float64, 1} + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood new all (l2+l22) +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nall5sol2flux1 = solve(prob, alg) +# nall5sol2flux1.estimated_ode_params[1]#6.54705 Particles{Float64, 1} +# nall5sol2flux2 = solve(prob, alg) +# nall5sol2flux2.estimated_ode_params[1]#6.6967 Particles{Float64, 1} +# nall5sol2flux3 = solve(prob, alg) +# nall5sol2flux3.estimated_ode_params[1]#6.47173 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nall5sol2flux11 = solve(prob, alg) +# nall5sol2flux11.estimated_ode_params[1]#6.2113 Particles{Float64, 1} +# nall5sol2flux22 = solve(prob, alg) +# nall5sol2flux22.estimated_ode_params[1]#6.10675 Particles{Float64, 1} +# nall5sol2flux33 = solve(prob, alg) +# nall5sol2flux33.estimated_ode_params[1]#6.11541 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nall5sol2flux111 = solve(prob, alg) +# nall5sol2flux111.estimated_ode_params[1]#6.35224 Particles{Float64, 1} +# nall5sol2flux222 = solve(prob, alg) +# nall5sol2flux222.estimated_ode_params[1]#6.40542 Particles{Float64, 1} +# nall5sol2flux333 = solve(prob, alg) +# nall5sol2flux333.estimated_ode_params[1]#6.44206 Particles{Float64, 1} + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood new 5* (new only l22 mod) +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n5sol2flux1 = solve(prob, alg) +# n5sol2flux1.estimated_ode_params[1]#7.05077 Particles{Float64, 1} +# n5sol2flux2 = solve(prob, alg) +# n5sol2flux2.estimated_ode_params[1]#7.07303 Particles{Float64, 1} +# n5sol2flux3 = solve(prob, alg) +# n5sol2flux3.estimated_ode_params[1]#5.10622 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n5sol2flux11 = solve(prob, alg) +# n5sol2flux11.estimated_ode_params[1]#7.39852 Particles{Float64, 1} +# n5sol2flux22 = solve(prob, alg) +# n5sol2flux22.estimated_ode_params[1]#7.30319 Particles{Float64, 1} +# n5sol2flux33 = solve(prob, alg) +# n5sol2flux33.estimated_ode_params[1]#6.73722 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n5sol2flux111 = solve(prob, alg) +# n5sol2flux111.estimated_ode_params[1]#7.15996 Particles{Float64, 1} +# n5sol2flux222 = solve(prob, alg) +# n5sol2flux222.estimated_ode_params[1]#7.02949 Particles{Float64, 1} +# n5sol2flux333 = solve(prob, alg) +# n5sol2flux333.estimated_ode_params[1]#6.9393 Particles{Float64, 1} + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood new +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nsol2flux1 = solve(prob, alg) +# nsol2flux1.estimated_ode_params[1] #5.82707 Particles{Float64, 1} +# nsol2flux2 = solve(prob, alg) +# nsol2flux2.estimated_ode_params[1] #4.81534 Particles{Float64, 1} +# nsol2flux3 = solve(prob, alg) +# nsol2flux3.estimated_ode_params[1] #5.52965 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nsol2flux11 = solve(prob, alg) +# nsol2flux11.estimated_ode_params[1] #7.04027 Particles{Float64, 1} +# nsol2flux22 = solve(prob, alg) +# nsol2flux22.estimated_ode_params[1] #7.17588 Particles{Float64, 1} +# nsol2flux33 = solve(prob, alg) +# nsol2flux33.estimated_ode_params[1] #6.94495 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nsol2flux111 = solve(prob, alg) +# nsol2flux111.estimated_ode_params[1] #6.06608 Particles{Float64, 1} +# nsol2flux222 = solve(prob, alg) +# nsol2flux222.estimated_ode_params[1] #6.84726 Particles{Float64, 1} +# nsol2flux333 = solve(prob, alg) +# nsol2flux333.estimated_ode_params[1] #6.83463 Particles{Float64, 1} + +# # ---------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux1 = solve(prob, alg) +# sol2flux1.estimated_ode_params[1] #6.71397 Particles{Float64, 1} 6.37604 Particles{Float64, 1} +# sol2flux2 = solve(prob, alg) +# sol2flux2.estimated_ode_params[1] #6.73509 Particles{Float64, 1} 6.21692 Particles{Float64, 1} +# sol2flux3 = solve(prob, alg) +# sol2flux3.estimated_ode_params[1] #6.65453 Particles{Float64, 1} 6.23153 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux11 = solve(prob, alg) +# sol2flux11.estimated_ode_params[1] #6.23443 Particles{Float64, 1} 6.30635 Particles{Float64, 1} +# sol2flux22 = solve(prob, alg) +# sol2flux22.estimated_ode_params[1] #6.18879 Particles{Float64, 1} 6.30099 Particles{Float64, 1} +# sol2flux33 = solve(prob, alg) +# sol2flux33.estimated_ode_params[1] #6.22773 Particles{Float64, 1} 6.30671 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux111 = solve(prob, alg) +# sol2flux111.estimated_ode_params[1] #6.15832 Particles{Float64, 1} 6.35453 Particles{Float64, 1} +# sol2flux222 = solve(prob, alg) +# sol2flux222.estimated_ode_params[1] #6.16968 Particles{Float64, 1}6.31125 Particles{Float64, 1} +# sol2flux333 = solve(prob, alg) +# sol2flux333.estimated_ode_params[1] #6.12466 Particles{Float64, 1} 6.26514 Particles{Float64, 1} + +# # ---------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, partial likelihood +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux1_p = solve(prob, alg) +# sol2flux1_p.estimated_ode_params[1] #5.74065 Particles{Float64, 1} #6.83683 Particles{Float64, 1} +# sol2flux2_p = solve(prob, alg) +# sol2flux2_p.estimated_ode_params[1] #9.82504 Particles{Float64, 1} #6.14568 Particles{Float64, 1} +# sol2flux3_p = solve(prob, alg) +# sol2flux3_p.estimated_ode_params[1] #5.75075 Particles{Float64, 1} #6.08579 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux11_p = solve(prob, alg) +# sol2flux11_p.estimated_ode_params[1] #6.19414 Particles{Float64, 1} #6.04621 Particles{Float64, 1} +# sol2flux22_p = solve(prob, alg) +# sol2flux22_p.estimated_ode_params[1] #6.15227 Particles{Float64, 1} #6.29086 Particles{Float64, 1} +# sol2flux33_p = solve(prob, alg) +# sol2flux33_p.estimated_ode_params[1] #6.19048 Particles{Float64, 1} #6.12516 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux111_p = solve(prob, alg) +# sol2flux111_p.estimated_ode_params[1] #6.51608 Particles{Float64, 1}# 6.42945Particles{Float64, 1} +# sol2flux222_p = solve(prob, alg) +# sol2flux222_p.estimated_ode_params[1] #6.4875 Particles{Float64, 1} # 6.44524Particles{Float64, 1} +# sol2flux333_p = solve(prob, alg) +# sol2flux333_p.estimated_ode_params[1] #6.51679 Particles{Float64, 1}# 6.43152Particles{Float64, 1} + +# # --------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood, dataset(1.0-2.0) +# # 25 points +# ta = range(1.0, tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux1 = solve(prob, alg) +# sol1flux1.estimated_ode_params[1] #6.35164 Particles{Float64, 1} +# sol1flux2 = solve(prob, alg) +# sol1flux2.estimated_ode_params[1] #6.30919 Particles{Float64, 1} +# sol1flux3 = solve(prob, alg) +# sol1flux3.estimated_ode_params[1] #6.33554 Particles{Float64, 1} + +# # 50 points +# ta = range(1.0, tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux11 = solve(prob, alg) +# sol1flux11.estimated_ode_params[1] #6.39769 Particles{Float64, 1} +# sol1flux22 = solve(prob, alg) +# sol1flux22.estimated_ode_params[1] #6.43924 Particles{Float64, 1} +# sol1flux33 = solve(prob, alg) +# sol1flux33.estimated_ode_params[1] #6.4697 Particles{Float64, 1} + +# # 100 points +# ta = range(1.0, tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux111 = solve(prob, alg) +# sol1flux111.estimated_ode_params[1] #6.27812 Particles{Float64, 1} +# sol1flux222 = solve(prob, alg) +# sol1flux222.estimated_ode_params[1] #6.19278 Particles{Float64, 1} +# sol1flux333 = solve(prob, alg) +# sol1flux333.estimated_ode_params[1] # 9.68244Particles{Float64, 1} (first try) # 6.23969 Particles{Float64, 1}(second try) + +# # ---------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, partial likelihood, dataset(1.0-2.0) +# # 25 points +# ta = range(1.0, tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux1_p = solve(prob, alg) +# sol1flux1_p.estimated_ode_params[1]#6.36269 Particles{Float64, 1} + +# sol1flux2_p = solve(prob, alg) +# sol1flux2_p.estimated_ode_params[1]#6.34685 Particles{Float64, 1} + +# sol1flux3_p = solve(prob, alg) +# sol1flux3_p.estimated_ode_params[1]#6.31421 Particles{Float64, 1} + +# # 50 points +# ta = range(1.0, tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux11_p = solve(prob, alg) +# sol1flux11_p.estimated_ode_params[1] #6.15725 Particles{Float64, 1} + +# sol1flux22_p = solve(prob, alg) +# sol1flux22_p.estimated_ode_params[1] #6.18145 Particles{Float64, 1} + +# sol1flux33_p = solve(prob, alg) +# sol1flux33_p.estimated_ode_params[1] #6.21905 Particles{Float64, 1} + +# # 100 points +# ta = range(1.0, tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux111_p = solve(prob, alg) +# sol1flux111_p.estimated_ode_params[1]#6.13481 Particles{Float64, 1} + +# sol1flux222_p = solve(prob, alg) +# sol1flux222_p.estimated_ode_params[1]#9.68555 Particles{Float64, 1} + +# sol1flux333_p = solve(prob, alg) +# sol1flux333_p.estimated_ode_params[1]#6.1477 Particles{Float64, 1} + +# # ----------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, partial likelihood, dataset(1-2), again but different density +# # 12 points +# ta = range(1.0, tspan[2], length = 12) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol3flux1_p = solve(prob, alg) +# sol3flux1_p.estimated_ode_params[1]#6.50048 Particles{Float64, 1} +# sol3flux2_p = solve(prob, alg) +# sol3flux2_p.estimated_ode_params[1]#6.57597 Particles{Float64, 1} +# sol3flux3_p = solve(prob, alg) +# sol3flux3_p.estimated_ode_params[1]#6.24487 Particles{Float64, 1} + +# # 25 points +# ta = range(1.0, tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol3flux11_p = solve(prob, alg) +# sol3flux11_p.estimated_ode_params[1]#6.53093 Particles{Float64, 1} + +# sol3flux22_p = solve(prob, alg) +# sol3flux22_p.estimated_ode_params[1]#6.32744 Particles{Float64, 1} + +# sol3flux33_p = solve(prob, alg) +# sol3flux33_p.estimated_ode_params[1]#6.49175 Particles{Float64, 1} + +# # 50 points +# ta = range(1.0, tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol3flux111_p = solve(prob, alg) +# sol3flux111_p.estimated_ode_params[1]#6.4455 Particles{Float64, 1} +# sol3flux222_p = solve(prob, alg) +# sol3flux222_p.estimated_ode_params[1]#6.40736 Particles{Float64, 1} +# sol3flux333_p = solve(prob, alg) +# sol3flux333_p.estimated_ode_params[1]#6.46214 Particles{Float64, 1} + +# # --------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, partial likelihood, dataset(0-1) +# # 25 points +# ta = range(tspan[1], 1.0, length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol0flux1_p = solve(prob, alg) +# sol0flux1_p.estimated_ode_params[1]#7.12625 Particles{Float64, 1} +# sol0flux2_p = solve(prob, alg) +# sol0flux2_p.estimated_ode_params[1]#8.40948 Particles{Float64, 1} +# sol0flux3_p = solve(prob, alg) +# sol0flux3_p.estimated_ode_params[1]#7.18768 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], 1.0, length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol0flux11_p = solve(prob, alg) +# sol0flux11_p.estimated_ode_params[1]#6.23707 Particles{Float64, 1} +# sol0flux22_p = solve(prob, alg) +# sol0flux22_p.estimated_ode_params[1]#6.09728 Particles{Float64, 1} +# sol0flux33_p = solve(prob, alg) +# sol0flux33_p.estimated_ode_params[1]#6.12971 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], 1.0, length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol0flux111_p = solve(prob, alg) +# sol0flux111_p.estimated_ode_params[1]#5.99039 Particles{Float64, 1} +# sol0flux222_p = solve(prob, alg) +# sol0flux222_p.estimated_ode_params[1]#5.89609 Particles{Float64, 1} +# sol0flux333_p = solve(prob, alg) +# sol0flux333_p.estimated_ode_params[1]#5.91923 Particles{Float64, 1} + +# # --------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood, dataset(1.0-2.0), Normal(12,5) distri prior +# # 25 points +# ta = range(1.0, tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 6.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f1 = solve(prob, alg) +# sol1f1.estimated_ode_params[1] +# # 10.9818Particles{Float64, 1} +# sol1f2 = solve(prob, alg) +# sol1f2.estimated_ode_params[1] +# # sol1f3 = solve(prob, alg) +# # sol1f3.estimated_ode_params[1] + +# # 50 points +# ta = range(1.0, tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 6.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f11 = solve(prob, alg) +# sol1f11.estimated_ode_params[1] +# sol1f22 = solve(prob, alg) +# sol1f22.estimated_ode_params[1] +# # sol1f33 = solve(prob, alg) +# # sol1f33.estimated_ode_params[1] + +# # 100 points +# ta = range(1.0, tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 6.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f111 = solve(prob, alg) +# sol1f111.estimated_ode_params[1] +# sol1f222 = solve(prob, alg) +# sol1f222.estimated_ode_params[1] +# # sol1f333 = solve(prob, alg) +# # sol1f333.estimated_ode_params[1] + +# # ---------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, partial likelihood, dataset(1.0-2.0), Normal(12,5) distri prior +# # 25 points +# ta = range(1.0, tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f1_p = solve(prob, alg) +# sol1f1_p.estimated_ode_params[1] +# sol1f2_p = solve(prob, alg) +# sol1f2_p.estimated_ode_params[1] +# sol1f3_p = solve(prob, alg) +# sol1f3_p.estimated_ode_params[1] + +# # 50 points +# ta = range(1.0, tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f11_p = solve(prob, alg) +# sol1f11_p.estimated_ode_params[1] +# sol1f22_p = solve(prob, alg) +# sol1f22_p.estimated_ode_params[1] +# sol1f33_p = solve(prob, alg) +# sol1f33_p.estimated_ode_params[1] + +# # 100 points +# ta = range(1.0, tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f111_p = solve(prob, alg) +# sol1f111_p.estimated_ode_params[1] +# sol1f222_p = solve(prob, alg) +# sol1f222_p.estimated_ode_params[1] +# sol1f333_p = solve(prob, alg) +# sol1f333_p.estimated_ode_params[1] + +# # ---------------------------------------------------------- + +# plot!(title = "9,2.5 50 training 2>full,1>partial") + +# p +# param1 +# # (lux chain) +# @prob mean(abs.(physsol2 .- sol3lux_pestim.ensemblesol[1])) < 8e-2 + +# # estimated parameters(lux chain) +# param1 = sol3lux_pestim.estimated_ode_params[1] +# @test abs(param1 - p) < abs(0.35 * p) + +# p +# param1 + +# # # my suggested Loss likelihood part +# # # + L2loss2(Tar, θ) +# # # My suggested extra loss function +# # function L2loss2(Tar::LogTargetDensity, θ) +# # f = Tar.prob.f + +# # # parameter estimation chosen or not +# # if Tar.extraparams > 0 +# # dataset = Tar.dataset + +# # # Timepoints to enforce Physics +# # dataset = Array(reduce(hcat, dataset)') +# # t = dataset[end, :] +# # û = dataset[1:(end - 1), :] + +# # ode_params = Tar.extraparams == 1 ? +# # θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : +# # θ[((length(θ) - Tar.extraparams) + 1):length(θ)] + +# # if length(û[:, 1]) == 1 +# # physsol = [f(û[:, i][1], +# # ode_params, +# # t[i]) +# # for i in 1:length(û[1, :])] +# # else +# # physsol = [f(û[:, i], +# # ode_params, +# # t[i]) +# # for i in 1:length(û[1, :])] +# # end +# # #form of NN output matrix output dim x n +# # deri_physsol = reduce(hcat, physsol) + +# # # OG deriv(basically gradient matching in case of an ODEFunction) +# # # in case of PDE or general ODE we would want to reduce residue of f(du,u,p,t) +# # # if length(û[:, 1]) == 1 +# # # deri_sol = [f(û[:, i][1], +# # # Tar.prob.p, +# # # t[i]) +# # # for i in 1:length(û[1, :])] +# # # else +# # # deri_sol = [f(û[:, i], +# # # Tar.prob.p, +# # # t[i]) +# # # for i in 1:length(û[1, :])] +# # # end +# # # deri_sol = reduce(hcat, deri_sol) +# # derivatives = calculate_derivatives(Tar.dataset) +# # deri_sol = reduce(hcat, derivatives) + +# # physlogprob = 0 +# # for i in 1:length(Tar.prob.u0) +# # # can add phystd[i] for u[i] +# # physlogprob += logpdf(MvNormal(deri_physsol[i, :], +# # LinearAlgebra.Diagonal(map(abs2, +# # Tar.l2std[i] .* +# # ones(length(deri_sol[i, :]))))), +# # deri_sol[i, :]) +# # end +# # return physlogprob +# # else +# # return 0 +# # end +# # end + +# # function calculate_derivatives(dataset) +# # x̂, time = dataset +# # num_points = length(x̂) + +# # # Initialize an array to store the derivative values. +# # derivatives = similar(x̂) + +# # for i in 2:(num_points - 1) +# # # Calculate the first-order derivative using central differences. +# # Δt_forward = time[i + 1] - time[i] +# # Δt_backward = time[i] - time[i - 1] + +# # derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) + +# # derivatives[i] = derivative +# # end + +# # # Derivatives at the endpoints can be calculated using forward or backward differences. +# # derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) +# # derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) + +# # return derivatives +# # end + +# size(dataset[1]) +# # Problem 1 with param estimation(flux,lux) +# # Normal +# # 6.20311 Particles{Float64, 1},6.21746Particles{Float64, 1} +# # better +# # 6.29093Particles{Float64, 1}, 6.27925Particles{Float64, 1} +# # Non ideal case +# # 6.14861Particles{Float64, 1}, +# sol2flux.estimated_ode_params +# sol2lux.estimated_ode_params[1] +# p +# size(sol3flux_pestim.ensemblesol[2]) +# plott = sol3flux_pestim.ensemblesol[1] +# using StatsPlots +# plotly() +# plot(t, sol3flux_pestim.ensemblesol[1]) + +# function calculate_derivatives(dataset) +# x̂, time = dataset +# num_points = length(x̂) + +# # Initialize an array to store the derivative values. +# derivatives = similar(x̂) + +# for i in 2:(num_points - 1) +# # Calculate the first-order derivative using central differences. +# Δt_forward = time[i + 1] - time[i] +# Δt_backward = time[i] - time[i - 1] + +# derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) + +# derivatives[i] = derivative +# end + +# # Derivatives at the endpoints can be calculated using forward or backward differences. +# derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) +# derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) + +# return derivatives +# end + +# # Example usage: +# # dataset = [x̂, time] +# derivatives = calculate_derivatives(dataset) +# dataset[1] +# # Access derivative values at specific time points as needed. + +# # # 9,0.5 +# # 0.09894916260292887 +# # 0.09870335436072103 +# # 0.08398556878067913 +# # 0.10109070099105527 +# # 0.09122683737517055 +# # 0.08614958011892977 +# # mean(abs.(x̂ .- meanscurve1)) #0.017112298305523976 +# # mean(abs.(physsol1 .- meanscurve1)) #0.004038636894341354 +# # # 9,4(little worse) +# # mean(abs.(x̂ .- meanscurve1))#0.01800876370000113 +# # mean(abs.(physsol1 .- meanscurve1))#0.007285681280600875 +# # # 30,30 +# # mean(abs.(x̂ .- meanscurve1)) #0.10599926120358046 +# # mean(abs.(physsol1 .- meanscurve1)) #0.10375554193397989 +# # # 30,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.10160824458252521 +# # mean(abs.(physsol1 .- meanscurve1)) #0.09999942538357891 + +# # # ------------------------------------------------normale +# # # 9,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.0333356493928835 +# # mean(abs.(physsol1 .- meanscurve1)) #0.02721733876400459 +# # # 9,4(little worse) +# # mean(abs.(x̂ .- meanscurve1)) #0.020734206709433347 +# # mean(abs.(physsol1 .- meanscurve1)) #0.012502850740700212 +# # # 30,30 +# # mean(abs.(x̂ .- meanscurve1)) #0.10615859683094729 +# # mean(abs.(physsol1 .- meanscurve1)) #0.10508141153722575 +# # # 30,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.10833514946031565 +# # mean(abs.(physsol1 .- meanscurve1)) #0.10668470203219232 + +# # # 9,0.5 +# # 10.158108285475553 +# # 10.207234384538026 +# # 10.215000657664852 +# # 10.213817644016174 +# # 13.380030074088719 +# # 13.348906350967326 + +# # 6.952731422892041 + +# # # All losses +# # 10.161478523326277 +# # # L2 losses 1 +# # 9.33312996960278 +# # # L2 losses 2 +# # 10.217417241370631 + +# # mean([fhsamples1[i][26] for i in 500:1000]) #6.245045767509431 +# # p #6.283185307179586 +# # # 9,4 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.212522300650451 +# # # 30,30 +# # mean([fhsamples1[i][23] for i in 500:1000]) #35.328636809737695 +# # # 30,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #35.232963812125654 + +# # # ---------------------------------------normale +# # # 9,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.547771572198114 +# # p #6.283185307179586 +# # # 9,4 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.158906185002702 +# # # 30,30 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.210400972620185 +# # # 30,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.153845019454522 + +# # # ----------------more dataset normale ----------------------------- +# # # 9,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.271141178216537 +# # p #6.283185307179586 +# # # 9,4 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.241144692919369 +# # # 30,30 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.124480447973127 +# # # 30,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.07838011629903 + +# # # 9,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.016551602015599295 +# # mean(abs.(physsol1 .- meanscurve1)) #0.0021488618484224245 +# # # 9,4(little worse) +# # mean(abs.(x̂ .- meanscurve1)) #0.017022725082640747 +# # mean(abs.(physsol1 .- meanscurve1)) #0.004339761917100232 +# # # 30,30 +# # mean(abs.(x̂ .- meanscurve1)) #0.09668785317864312 +# # mean(abs.(physsol1 .- meanscurve1)) #0.09430712337543362 +# # # 30,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.09958118358974392 +# # mean(abs.(physsol1 .- meanscurve1)) #0.09717454226368502 + +# # # ----------------more dataset special ----------------------------- +# # # 9,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.284355334485365 +# # p #6.283185307179586 +# # # 9,4 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.259238106698602 +# # # 30,30 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.139808934336987 +# # # 30,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.03921327641226 + +# # # 9,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.016627231605546876 +# # mean(abs.(physsol1 .- meanscurve1)) #0.0020311429130039564 +# # # 9,4(little worse) +# # mean(abs.(x̂ .- meanscurve1)) #0.016650324577507352 +# # mean(abs.(physsol1 .- meanscurve1)) #0.0027537543411154677 +# # # 30,30 +# # mean(abs.(x̂ .- meanscurve1)) #0.09713187937270151 +# # mean(abs.(physsol1 .- meanscurve1)) #0.09317278450371556 +# # # 30,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.09550234866855814 +# # mean(abs.(physsol1 .- meanscurve1)) #0.09317278450371556 + +# # using Plots, StatsPlots +# # plotly() + +# # --------------------------------------------------------- +# # # # Distribution abstract in wrapper, dataset Float64 +# # # 268.651 s (206393690 allocations: 388.71 GiB) +# # # 318.170551 seconds (206.29 M allocations: 388.453 GiB, 20.83% gc time) + +# # # # Above with dataset Real subtype +# # # 326.201 s (206327409 allocations: 388.42 GiB) +# # # 363.189370 seconds (206.25 M allocations: 387.975 GiB, 15.77% gc time) +# # # 306.171 s (206321277 allocations: 388.55 GiB) +# # # 356.180699 seconds (206.43 M allocations: 388.361 GiB, 13.77% gc time) + +# # # # Above with dataset AbstractFloat subtype +# # # 290.751187 seconds (205.94 M allocations: 387.955 GiB, 12.92% gc time) +# # # 296.319815 seconds (206.38 M allocations: 388.730 GiB, 12.69% gc time) + +# # # # ODEProblem float64 dtaset and vector distri inside +# # # 273.169 s (206128318 allocations: 388.40 GiB) +# # # 274.059531 seconds (205.91 M allocations: 387.953 GiB, 12.77% gc time) + +# # # # Dataset float64 inside and vector distri outsude +# # # 333.603 s (206251143 allocations: 388.41 GiB) +# # # 373.377222 seconds (206.11 M allocations: 387.968 GiB, 13.25% gc time) +# # # 359.745 s (206348301 allocations: 388.41 GiB) +# # # 357.813114 seconds (206.31 M allocations: 388.354 GiB, 13.54% gc time) + +# # # # Dataset float64 inside and vector distri inside +# # # 326.437 s (206253571 allocations: 388.41 GiB) +# # # 290.334083 seconds (205.92 M allocations: 387.954 GiB, 13.82% gc time) + +# # # # current setting +# # # 451.304 s (206476927 allocations: 388.43 GiB) +# # # 384.532732 seconds (206.22 M allocations: 387.976 GiB, 13.17% gc time) +# # # 310.223 s (206332558 allocations: 388.63 GiB) +# # # 344.243889 seconds (206.34 M allocations: 388.409 GiB, 13.84% gc time) +# # # 357.457737 seconds (206.66 M allocations: 389.064 GiB, 18.16% gc time) + +# # # # shit setup +# # # 325.595 s (206283732 allocations: 388.41 GiB) +# # # 334.248753 seconds (206.06 M allocations: 387.964 GiB, 12.60% gc time) +# # # 326.011 s (206370857 allocations: 388.56 GiB) +# # # 327.203339 seconds (206.29 M allocations: 388.405 GiB, 12.92% gc time) + +# # # # in wrapper Distribution prior, insiade FLOAT64 DATASET +# # # 325.158167 seconds (205.97 M allocations: 387.958 GiB, 15.07% gc time) +# # # 429.536 s (206476324 allocations: 388.43 GiB) +# # # 527.364 s (206740343 allocations: 388.58 GiB) + +# # # # wrapper Distribtuion, inside Float64 +# # # 326.017 s (206037971 allocations: 387.96 GiB) +# # # 347.424730 seconds (206.45 M allocations: 388.532 GiB, 12.92% gc time) + +# # # 439.047568 seconds (284.24 M allocations: 392.598 GiB, 15.25% gc time, 14.36% compilation time: 0% of which was recompilation) +# # # 375.472142 seconds (206.40 M allocations: 388.529 GiB, 14.93% gc time) +# # # 374.888820 seconds (206.34 M allocations: 388.346 GiB, 14.09% gc time) +# # # 363.719611 seconds (206.39 M allocations: 388.581 GiB, 15.08% gc time) +# # # # inside Distribtion, instide Float64 +# # # 310.238 s (206324249 allocations: 388.53 GiB) +# # # 308.991494 seconds (206.34 M allocations: 388.549 GiB, 14.01% gc time) +# # # 337.442 s (206280712 allocations: 388.36 GiB) +# # # 299.983096 seconds (206.29 M allocations: 388.512 GiB, 17.14% gc time) + +# # # 394.924357 seconds (206.27 M allocations: 388.337 GiB, 23.68% gc time) +# # # 438.204179 seconds (206.39 M allocations: 388.470 GiB, 23.84% gc time) +# # # 376.626914 seconds (206.46 M allocations: 388.693 GiB, 18.72% gc time) +# # # 286.863795 seconds (206.14 M allocations: 388.370 GiB, 18.80% gc time) +# # # 285.556929 seconds (206.22 M allocations: 388.371 GiB, 17.04% gc time) +# # # 291.471662 seconds (205.96 M allocations: 388.068 GiB, 19.85% gc time) + +# # # 495.814341 seconds (284.62 M allocations: 392.622 GiB, 12.56% gc time, 10.96% compilation time: 0% of which was recompilation) +# # # 361.530617 seconds (206.36 M allocations: 388.526 GiB, 14.98% gc time) +# # # 348.576065 seconds (206.22 M allocations: 388.337 GiB, 15.01% gc time) +# # # 374.575609 seconds (206.45 M allocations: 388.586 GiB, 14.65% gc time) +# # # 314.223008 seconds (206.23 M allocations: 388.411 GiB, 14.63% gc time) + +# # PROBLEM-3 LOTKA VOLTERRA EXAMPLE [WIP] (WITH PARAMETER ESTIMATION)(will be put in tutorial page) +# function lotka_volterra(u, p, t) +# # Model parameters. +# α, β, γ, δ = p +# # Current state. +# x, y = u + +# # Evaluate differential equations. +# dx = (α - β * y) * x # prey +# dy = (δ * x - γ) * y # predator + +# return [dx, dy] +# end + +# u0 = [1.0, 1.0] +# p = [1.5, 1.0, 3.0, 1.0] +# tspan = (0.0, 6.0) +# prob = ODEProblem(lotka_volterra, u0, tspan, p) +# solution = solve(prob, Tsit5(); saveat = 0.05) + +# as = reduce(hcat, solution.u) +# as[1, :] +# # Plot simulation. +# time = solution.t +# u = hcat(solution.u...) +# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct +# x = u[1, :] + 0.5 * randn(length(u[1, :])) +# y = u[2, :] + 0.5 * randn(length(u[1, :])) +# dataset = [x[1:50], y[1:50], time[1:50]] +# # scatter!(time, [x, y]) +# # scatter!(dataset[3], [dataset[2], dataset[1]]) + +# # NN has 2 outputs as u -> [dx,dy] +# chainlux1 = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), +# Lux.Dense(6, 2)) +# chainflux1 = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) + +# # fh_mcmc_chainflux1, fhsamplesflux1, fhstatsflux1 = ahmc_bayesian_pinn_ode(prob, chainflux1, +# # dataset = dataset, +# # draw_samples = 1000, +# # l2std = [ +# # 0.05, +# # 0.05, +# # ], +# # phystd = [ +# # 0.05, +# # 0.05, +# # ], +# # priorsNNw = (0.0, +# # + +# # 3.0)) + +# # check if NN output is more than 1 +# # numoutput = size(luxar[1])[1] +# # if numoutput > 1 +# # # Initialize a vector to store the separated outputs for each output dimension +# # output_matrices = [Vector{Vector{Float32}}() for _ in 1:numoutput] + +# # # Loop through each element in the `as` vector +# # for element in as +# # for i in 1:numoutput +# # push!(output_matrices[i], element[i, :]) # Append the i-th output (i-th row) to the i-th output_matrices +# # end +# # end + +# # ensemblecurves = Vector{}[] +# # for r in 1:numoutput +# # br = hcat(output_matrices[r]...)' +# # ensemblecurve = prob.u0[r] .+ +# # [Particles(br[:, i]) for i in 1:length(t)] .* +# # (t .- prob.tspan[1]) +# # push!(ensemblecurves, ensemblecurve) +# # end + +# # else +# # # ensemblecurve = prob.u0 .+ +# # # [Particles(reduce(vcat, luxar)[:, i]) for i in 1:length(t)] .* +# # # (t .- prob.tspan[1]) +# # print("yuh") +# # end + +# # fhsamplesflux2 +# # nnparams = length(init1) +# # estimnnparams = [Particles(reduce(hcat, fhsamplesflux2)[i, :]) for i in 1:nnparams] +# # ninv=4 +# # estimated_params = [Particles(reduce(hcat, fhsamplesflux2[(end - ninv + 1):end])[i, :]) +# # for i in (nnparams + 1):(nnparams + ninv)] +# # output_matrices[r] +# # br = hcat(output_matrices[r]...)' + +# # br[:, 1] + +# # [Particles(br[:, i]) for i in 1:length(t)] +# # prob.u0 +# # [Particles(br[:, i]) for i in 1:length(t)] .* +# # (t .- prob.tspan[1]) + +# # ensemblecurve = prob.u0[r] .+ +# # [Particles(br[:, i]) for i in 1:length(t)] .* +# # (t .- prob.tspan[1]) +# # push!(ensemblecurves, ensemblecurve) + +# using StatsPlots +# plotly() +# plot(t, ensemblecurve) +# plot(t, ensemblecurves[1]) +# plot!(t, ensemblecurves[2]) +# ensemblecurve +# ensemblecurves[1] +# fh_mcmc_chainflux2, fhsamplesflux2, fhstatsflux2 = ahmc_bayesian_pinn_ode(prob, chainflux1, +# dataset = dataset, +# draw_samples = 1000, +# l2std = [ +# 0.05, +# 0.05, +# ], +# phystd = [ +# 0.05, +# 0.05, +# ], +# priorsNNw = (0.0, +# 3.0), +# param = [ +# Normal(1.5, +# 0.5), +# Normal(1.2, +# 0.5), +# Normal(3.3, +# 0.5), +# Normal(1.4, +# 0.5), +# ], progress = true) + +# alg = NeuralPDE.BNNODE(chainflux1, +# dataset = dataset, +# draw_samples = 1000, +# l2std = [ +# 0.05, +# 0.05, +# ], +# phystd = [ +# 0.05, +# 0.05, +# ], +# priorsNNw = (0.0, +# 3.0), +# param = [ +# Normal(4.5, +# 5), +# Normal(7, +# 2), +# Normal(5, +# 2), +# Normal(-4, +# 6), +# ], +# n_leapfrog = 30, progress = true) + +# sol3flux_pestim = solve(prob, alg) + +# # OG PARAM VALUES +# [1.5, 1.0, 3.0, 1.0] +# # less +# # [1.34, 7.51, 2.54, -2.55] +# # better +# # [1.48, 0.993, 2.77, 0.954] + +# sol3flux_pestim.es +# sol3flux_pestim.estimated_ode_params +# # fh_mcmc_chainlux1, fhsampleslux1, fhstatslux1 = ahmc_bayesian_pinn_ode(prob, chainlux1, +# # dataset = dataset, +# # draw_samples = 1000, +# # l2std = [0.05, 0.05], +# # phystd = [ +# # 0.05, +# # 0.05, +# # ], +# # priorsNNw = (0.0, +# # 3.0)) + +# # fh_mcmc_chainlux2, fhsampleslux2, fhstatslux2 = ahmc_bayesian_pinn_ode(prob, chainlux1, +# # dataset = dataset, +# # draw_samples = 1000, +# # l2std = [0.05, 0.05], +# # phystd = [ +# # 0.05, +# # 0.05, +# # ], +# # priorsNNw = (0.0, +# # 3.0), +# # param = [ +# # Normal(1.5, 0.5), +# # Normal(1.2, 0.5), +# # Normal(3.3, 0.5), +# # Normal(1.4, 0.5), +# # ]) + +# init1, re1 = destructure(chainflux1) +# θinit, st = Lux.setup(Random.default_rng(), chainlux1) +# # PLOT testing points +# t = time +# p = prob.p +# collect(Float64, vcat(ComponentArrays.ComponentArray(θinit))) +# collect(Float64, ComponentArrays.ComponentArray(θinit)) +# # Mean of last 1000 sampled parameter's curves(flux and lux chains)[Ensemble predictions] +# out = re1.([fhsamplesflux1[i][1:68] for i in 500:1000]) +# yu = [out[i](t') for i in eachindex(out)] + +# function getensemble(yu, num_models) +# num_rows, num_cols = size(yu[1]) +# row_means = zeros(Float32, num_rows, num_cols) +# for i in 1:num_models +# row_means .+= yu[i] +# end +# row_means ./ num_models +# end +# fluxmean = getensemble(yu, length(out)) +# meanscurve1_1 = prob.u0 .+ (t' .- prob.tspan[1]) .* fluxmean +# mean(abs.(u .- meanscurve1_1)) + +# plot!(t, physsol1) +# @test mean(abs2.(x̂ .- meanscurve1_1)) < 2e-2 +# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 +# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 +# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 + +# out = re1.([fhsamplesflux2[i][1:68] for i in 500:1000]) +# yu = collect(out[i](t') for i in eachindex(out)) +# fluxmean = getensemble(yu, length(out)) +# meanscurve1_2 = prob.u0 .+ (t' .- prob.tspan[1]) .* fluxmean +# mean(abs.(u .- meanscurve1_2)) + +# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 +# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 +# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 +# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 + +# θ = [vector_to_parameters(fhsampleslux1[i][1:(end - 4)], θinit) for i in 500:1000] +# luxar = [chainlux1(t', θ[i], st)[1] for i in 1:500] +# luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] +# meanscurve2_1 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean + +# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 +# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 +# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 +# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 + +# θ = [vector_to_parameters(fhsampleslux2[i][1:(end - 4)], θinit) for i in 500:1000] +# luxar = [chainlux1(t', θ[i], st)[1] for i in 1:500] +# luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] +# meanscurve2_2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean + +# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 +# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 +# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 +# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 + +# # # ESTIMATED ODE PARAMETERS (NN1 AND NN2) +# @test abs(p - mean([fhsamplesflux2[i][69] for i in 500:1000])) < 0.1 * p[1] +# @test abs(p - mean([fhsampleslux2[i][69] for i in 500:1000])) < 0.2 * p[1] + +# # @test abs(p - mean([fhsamplesflux2[i][70] for i in 500:1000])) < 0.1 * p[2] +# # @test abs(p - mean([fhsampleslux2[i][70] for i in 500:1000])) < 0.2 * p[2] + +# # @test abs(p - mean([fhsamplesflux2[i][71] for i in 500:1000])) < 0.1 * p[3] +# # @test abs(p - mean([fhsampleslux2[i][71] for i in 500:1000])) < 0.2 * p[3] + +# # @test abs(p - mean([fhsamplesflux2[i][72] for i in 500:1000])) < 0.1 * p[4] +# # @test abs(p - mean([fhsampleslux2[i][72] for i in 500:1000])) < 0.2 * p[4] + +# # fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainlux1, +# # dataset = dataset, +# # draw_samples = 1000, +# # l2std = [0.05, 0.05], +# # phystd = [0.05, 0.05], +# # priorsNNw = (0.0, 3.0), +# # param = [ +# # Normal(1.5, 0.5), +# # Normal(1.2, 0.5), +# # Normal(3.3, 0.5), +# # Normal(1.4, 0.5), +# # ], autodiff = true) + +# # fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainlux1, +# # dataset = dataset, +# # draw_samples = 1000, +# # l2std = [0.05, 0.05], +# # phystd = [0.05, 0.05], +# # priorsNNw = (0.0, 3.0), +# # param = [ +# # Normal(1.5, 0.5), +# # Normal(1.2, 0.5), +# # Normal(3.3, 0.5), +# # Normal(1.4, 0.5), +# # ], nchains = 2) + +# # NOTES (WILL CLEAR LATER) +# # -------------------------------------------------------------------------------------------- +# # Hamiltonian energy must be lowest(more paramters the better is it to map onto them) +# # full better than L2 and phy individual(test) +# # in mergephys more points after training points is better from 20->40 +# # does consecutive runs bceome better? why?(plot 172)(same chain maybe) +# # does density of points in timespan matter dataset vs internal timespan?(plot 172)(100+0.01) +# # when training from 0-1 and phys from 1-5 with 1/150 simple nn slow,but bigger nn faster decrease in Hmailtonian +# # bigger time interval more curves to adapt to only more parameters adapt to that, better NN architecture +# # higher order logproblems solve better +# # repl up up are same instances? but reexecute calls are new? + +# #Compare results against paper example +# # Lux chains support (DONE) +# # fix predictions for odes depending upon 1,p in f(u,p,t)(DONE) +# # lotka volterra learn curve beyond l2 losses(L2 losses determine accuracy of parameters)(parameters cant run free ∴ L2 interval only) +# # check if prameters estimation works(YES) +# # lotka volterra parameters estimate (DONE) + +# using NeuralPDE, Lux, Flux, Optimization, OptimizationOptimJL +# import ModelingToolkit: Interval +# using Plots, StatsPlots +# plotly() +# # Profile.init() + +# @parameters x y +# @variables u(..) +# Dxx = Differential(x)^2 +# Dyy = Differential(y)^2 + +# # 2D PDE +# eq = Dxx(u(x, y)) + Dyy(u(x, y)) ~ -sin(pi * x) * sin(pi * y) + +# # Boundary conditions +# bcs = [u(0, y) ~ 0.0, u(1, y) ~ 0.0, +# u(x, 0) ~ 0.0, u(x, 1) ~ 0.0] +# # Space and time domains +# domains = [x ∈ Interval(0.0, 1.0), +# y ∈ Interval(0.0, 1.0)] + +# # Neural network +# dim = 2 # number of dimensions +# chain = Flux.Chain(Flux.Dense(dim, 16, Lux.σ), Flux.Dense(16, 16, Lux.σ), Flux.Dense(16, 1)) +# θ, re = destructure(chain) +# # Discretization +# dx = 0.05 +# discretization = PhysicsInformedNN(chain, GridTraining(dx)) + +# @named pde_system = PDESystem(eq, bcs, domains, [x, y], [u(x, y)]) + +# pinnrep = symbolic_discretize(pde_system, discretization) +# typeof(pinnrep.phi) +# typeof(pinnrep.phi) +# typeof(re) +# pinnrep.phi([1, 2], θ) + +# typeof(θ) + +# print(pinnrep) +# pinnrep.eqs +# pinnrep.bcs +# pinnrep.domains +# pinnrep.eq_params +# pinnrep.defaults +# print(pinnrep.default_p) +# pinnrep.param_estim +# print(pinnrep.additional_loss) +# pinnrep.adaloss +# pinnrep.depvars +# pinnrep.indvars +# pinnrep.dict_depvar_input +# pinnrep.dict_depvars +# pinnrep.dict_indvars +# print(pinnrep.logger) +# pinnrep.multioutput +# pinnrep.iteration +# pinnrep.init_params +# pinnrep.flat_init_params +# pinnrep.phi +# pinnrep.derivative +# pinnrep.strategy +# pinnrep.pde_indvars +# pinnrep.bc_indvars +# pinnrep.pde_integration_vars +# pinnrep.bc_integration_vars +# pinnrep.integral +# pinnrep.symbolic_pde_loss_functions +# pinnrep.symbolic_bc_loss_functions +# pinnrep.loss_functions + +# # = discretize(pde_system, discretization) +# prob = symbolic_discretize(pde_system, discretization) +# # "The boundary condition loss functions" +# sum([prob.loss_functions.bc_loss_functions[i](θ) for i in eachindex(1:4)]) +# sum([prob.loss_functions.pde_loss_functions[i](θ) for i in eachindex(1)]) + +# prob.loss_functions.full_loss_function(θ, 32) + +# prob.loss_functions.bc_loss_functions[1](θ) + +# prob.loss_functions.bc_loss_functions +# prob.loss_functions.full_loss_function +# prob.loss_functions.additional_loss_function +# prob.loss_functions.pde_loss_functions + +# 1.3953060473003345 + 1.378102161087438 + 1.395376727128639 + 1.3783868705075002 + +# 0.22674532775196876 +# # "The PDE loss functions" +# prob.loss_functions.pde_loss_functions +# prob.loss_functions.pde_loss_functions[1](θ) +# # "The full loss function, combining the PDE and boundary condition loss functions.This is the loss function that is used by the optimizer." +# prob.loss_functions.full_loss_function(θ, nothing) +# prob.loss_functions.full_loss_function(θ, 423423) + +# # "The wrapped `additional_loss`, as pieced together for the optimizer." +# prob.loss_functions.additional_loss_function +# # "The pre-data version of the PDE loss function" +# prob.loss_functions.datafree_pde_loss_functions +# # "The pre-data version of the BC loss function" +# prob.loss_functions.datafree_bc_loss_functions + +# using Random +# θ, st = Lux.setup(Random.default_rng(), chain) +# #Optimizer +# opt = OptimizationOptimJL.BFGS() + +# #Callback function +# callback = function (p, l) +# println("Current loss is: $l") +# return false +# end + +# res = Optimization.solve(prob, opt, callback = callback, maxiters = 1000) +# phi = discretization.phi + +# # ------------------------------------------------ +# using NeuralPDE, Lux, ModelingToolkit, Optimization, OptimizationOptimJL, OrdinaryDiffEq, +# Plots +# import ModelingToolkit: Interval, infimum, supremum +# @parameters t, σ_, β, ρ +# @variables x(..), y(..), z(..) +# Dt = Differential(t) +# eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), +# Dt(y(t)) ~ x(t) * (ρ - z(t)) - y(t), +# Dt(z(t)) ~ x(t) * y(t) - β * z(t)] + +# bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] +# domains = [t ∈ Interval(0.0, 1.0)] +# dt = 0.01 + +# input_ = length(domains) +# n = 8 +# chain1 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, 1)) +# chain2 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, 1)) +# chain3 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, 1)) + +# function lorenz!(du, u, p, t) +# du[1] = 10.0 * (u[2] - u[1]) +# du[2] = u[1] * (28.0 - u[3]) - u[2] +# du[3] = u[1] * u[2] - (8 / 3) * u[3] +# end + +# u0 = [1.0; 0.0; 0.0] +# tspan = (0.0, 1.0) +# prob = ODEProblem(lorenz!, u0, tspan) +# sol = solve(prob, Tsit5(), dt = 0.1) +# ts = [infimum(d.domain):dt:supremum(d.domain) for d in domains][1] +# function getData(sol) +# data = [] +# us = hcat(sol(ts).u...) +# ts_ = hcat(sol(ts).t...) +# return [us, ts_] +# end +# data = getData(sol) + +# (u_, t_) = data +# len = length(data[2]) + +# depvars = [:x, :y, :z] +# function additional_loss(phi, θ, p) +# return sum(sum(abs2, phi[i](t_, θ[depvars[i]]) .- u_[[i], :]) / len for i in 1:1:3) +# end + +# discretization = NeuralPDE.PhysicsInformedNN([chain1, chain2, chain3], +# NeuralPDE.GridTraining(dt), +# param_estim = false, +# additional_loss = additional_loss) +# @named pde_system = PDESystem(eqs, bcs, domains, [t], [x(t), y(t), z(t)], [σ_, ρ, β], +# defaults = Dict([p .=> 1.0 for p in [σ_, ρ, β]])) +# prob = NeuralPDE.discretize(pde_system, discretization) +# callback = function (p, l) +# println("Current loss is: $l") +# return false +# end +# res = Optimization.solve(prob, BFGS(); callback = callback, maxiters = 5000) +# p_ = res.u[(end - 2):end] # p_ = [9.93, 28.002, 2.667] + +# minimizers = [res.u.depvar[depvars[i]] for i in 1:3] +# ts = [infimum(d.domain):(dt / 10):supremum(d.domain) for d in domains][1] +# u_predict = [[discretization.phi[i]([t], minimizers[i])[1] for t in ts] for i in 1:3] +# plot(sol) +# plot!(ts, u_predict, label = ["x(t)" "y(t)" "z(t)"]) + +# discretization.multioutput +# discretization.chain +# discretization.strategy +# discretization.init_params +# discretization.phi +# discretization.derivative +# discretization.param_estim +# discretization.additional_loss +# discretization.adaptive_loss +# discretization.logger +# discretization.log_options +# discretization.iteration +# discretization.self_increment +# discretization.multioutput +# discretization.kwargs + +# struct BNNODE1{P <: Vector{<:Distribution}} +# chain::Any +# Kernel::Any +# draw_samples::UInt32 +# priorsNNw::Tuple{Float64, Float64} +# param::P +# l2std::Vector{Float64} +# phystd::Vector{Float64} + +# function BNNODE1(chain, Kernel; draw_samples = 2000, priorsNNw = (0.0, 3.0), param = [], +# l2std = [0.05], phystd = [0.05]) +# BNNODE1(chain, Kernel, draw_samples, priorsNNw, param, l2std, phystd) +# end +# end + +# struct BNNODE3{C, K, P <: Union{Any, Vector{<:Distribution}}} +# chain::C +# Kernel::K +# draw_samples::UInt32 +# priorsNNw::Tuple{Float64, Float64} +# param::P +# l2std::Vector{Float64} +# phystd::Vector{Float64} + +# function BNNODE3(chain, Kernel; draw_samples = 2000, priorsNNw = (0.0, 3.0), param = [], +# l2std = [0.05], phystd = [0.05]) +# new{typeof(chain), typeof(Kernel), typeof(param)}(chain, Kernel, draw_samples, +# priorsNNw, param, l2std, phystd) +# end +# end +# linear_analytic = (u0, p, t) -> u0 + sin(2 * π * t) / (2 * π) +# linear = (u, p, t) -> cos(2 * π * t) +# tspan = (0.0, 2.0) +# u0 = 0.0 +# prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan) + +# ta = range(tspan[1], tspan[2], length = 300) +# u = [linear_analytic(u0, nothing, ti) for ti in ta] +# sol1 = solve(prob, Tsit5()) + +# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct +# x̂ = collect(Float64, Array(u) + 0.02 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂[1:100], time[1:100]] + +# # Call BPINN, create chain +# chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) +# chainlux = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) +# HMC +# solve(prob, BNNODE(chainflux, HMC)) +# BNNODE1(chainflux, HMC, 2000) + +# draw_samples = 2000 +# priorsNNw = (0.0, 3.0) +# param = [] +# l2std = [0.05] +# phystd = [0.05] +# @time BNNODE3(chainflux, HMC, draw_samples = 2000, priorsNNw = (0.0, 3.0), +# param = [nothing], +# l2std = [0.05], phystd = [0.05]) +# typeof(Nothing) <: Vector{<:Distribution} +# Nothing <: Distribution +# {UnionAll} <: Distribution +# @time [Nothing] +# typeof([Nothing]) +# @time [1] + +# function test1(sum; c = 23, d = 32) +# return sum + c + d +# end +# function test(a, b; c, d) +# return test1(a + b, c, d) +# end + +# test(2, 2) + +# struct BNNODE3{C, K, P <: Union{Vector{Nothing}, Vector{<:Distribution}}} +# chain::C +# Kernel::K +# draw_samples::Int64 +# priorsNNw::Tuple{Float64, Float64} +# param::P +# l2std::Vector{Float64} +# phystd::Vector{Float64} + +# function BNNODE3(chain, Kernel; draw_samples, +# priorsNNw, param = [nothing], l2std, phystd) +# new{typeof(chain), typeof(Kernel), typeof(param)}(chain, +# Kernel, +# draw_samples, +# priorsNNw, +# param, l2std, +# phystd) +# end +# end + +# function solve1(prob::DiffEqBase.AbstractODEProblem, alg::BNNODE3; +# dataset = [nothing], dt = 1 / 20.0, +# init_params = nothing, nchains = 1, +# autodiff = false, Integrator = Leapfrog, +# Adaptor = StanHMCAdaptor, targetacceptancerate = 0.8, +# Metric = DiagEuclideanMetric, jitter_rate = 3.0, +# tempering_rate = 3.0, max_depth = 10, Δ_max = 1000, +# n_leapfrog = 10, δ = 0.65, λ = 0.3, progress = true, +# verbose = false) +# chain = alg.chain +# l2std = alg.l2std +# phystd = alg.phystd +# priorsNNw = alg.priorsNNw +# Kernel = alg.Kernel +# draw_samples = alg.draw_samples + +# param = alg.param == [nothing] ? [] : alg.param +# mcmcchain, samples, statistics = ahmc_bayesian_pinn_ode(prob, chain, dataset = dataset, +# draw_samples = draw_samples, +# init_params = init_params, +# physdt = dt, l2std = l2std, +# phystd = phystd, +# priorsNNw = priorsNNw, +# param = param, +# nchains = nchains, +# autodiff = autodiff, +# Kernel = Kernel, +# Integrator = Integrator, +# Adaptor = Adaptor, +# targetacceptancerate = targetacceptancerate, +# Metric = Metric, +# jitter_rate = jitter_rate, +# tempering_rate = tempering_rate, +# max_depth = max_depth, +# Δ_max = Δ_max, +# n_leapfrog = n_leapfrog, δ = δ, +# λ = λ, progress = progress, +# verbose = verbose) +# end + +# linear_analytic = (u0, p, t) -> u0 + sin(2 * π * t) / (2 * π) +# linear = (u, p, t) -> cos(2 * π * t) +# tspan = (0.0, 2.0) +# u0 = 0.0 +# prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan) + +# ta = range(tspan[1], tspan[2], length = 300) +# u = [linear_analytic(u0, nothing, ti) for ti in ta] +# # sol1 = solve(prob, Tsit5()) + +# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct +# x̂ = collect(Float64, Array(u) + 0.02 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂[1:100], time[1:100]] + +# # Call BPINN, create chain +# chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) +# chainlux = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) +# HMC + +# solve1(prob, a) +# a = BNNODE3(chainflux, HMC, draw_samples = 2000, +# priorsNNw = (0.0, 3.0), +# l2std = [0.05], phystd = [0.05]) + +# Define Lotka-Volterra model. +function lotka_volterra1(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end + +u0 = [1.0, 1.0] +p = [1.5, 1.0, 3.0, 1.0] +tspan = (0.0, 6.0) +prob = ODEProblem(lotka_volterra1, u0, tspan, p) +solution = solve(prob, Tsit5(); saveat = 0.05) + +as = reduce(hcat, solution.u) +as[1, :] +# Plot simulation. +time = solution.t +u = hcat(solution.u...) +# BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct +x = u[1, :] + 0.5 * randn(length(u[1, :])) +y = u[2, :] + 0.5 * randn(length(u[1, :])) +dataset = [x[1:50], y[1:50], time[1:50]] +# scatter!(time, [x, y]) +# scatter!(dataset[3], [dataset[2], dataset[1]]) + +# NN has 2 outputs as u -> [dx,dy] +chainlux1 = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), + Lux.Dense(6, 2)) +chainflux1 = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) + +fh_mcmc_chainflux1, fhsamplesflux1, fhstatsflux1 = ahmc_bayesian_pinn_ode(prob, chainflux1, + dataset = dataset, + draw_samples = 1000, + l2std = [ + 0.05, + 0.05, + ], + phystd = [ + 0.05, + 0.05, + ], + priorsNNw = (0.0, 3.0), progress = true) +ahmc_bayesian_pinn_ode(prob, chainflux1, + dataset = dataset, + draw_samples = 1000, + l2std = [ + 0.05, + 0.05, + ], + phystd = [ + 0.05, + 0.05, + ], + priorsNNw = (0.0, 3.0), progress = true) + +# 2×171 Matrix{Float64}: +# -0.5 -0.518956 -0.529639 … -1.00266 -1.01049 +# 2.0 1.97109 1.92747 0.42619 0.396335 + +# 2-element Vector{Float64}: +# -119451.94949911036 +# -128543.23714618056 + +# alg = NeuralPDE.BNNODE(chainflux1, +# dataset = dataset, +# draw_samples = 1000, +# l2std = [ +# 0.05, +# 0.05, +# ], +# phystd = [ +# 0.05, +# 0.05, +# ], +# priorsNNw = (0.0, +# 3.0), +# param = [ +# Normal(4.5, +# 5), +# Normal(7, +# 2), +# Normal(5, +# 2), +# Normal(-4, +# 6), +# ], +# n_leapfrog = 30, progress = true) + +# sol3flux_pestim = solve(prob, alg) + +# ---------------------------------------------- +# original paper implementation +# 25 points +run1 #7.70593 Particles{Float64, 1} +run2 #6.66347 Particles{Float64, 1} +run3 #6.84827 Particles{Float64, 1} + +# 50 points +run1 #7.83577 Particles{Float64, 1} +run2 #6.49477 Particles{Float64, 1} +run3 #6.47421 Particles{Float64, 1} + +# 100 points +run1 #5.96604 Particles{Float64, 1} +run2 #6.05432 Particles{Float64, 1} +run3 #6.08856 Particles{Float64, 1} + +# Full likelihood(uses total variation regularized differentiation) +# 25 points +run1 #6.41722 Particles{Float64, 1} +run2 #6.42782 Particles{Float64, 1} +run3 #6.42782 Particles{Float64, 1} + +# 50 points +run1 #5.71268 Particles{Float64, 1} +run2 #5.74599 Particles{Float64, 1} +run3 #5.74599 Particles{Float64, 1} + +# 100 points +run1 #6.59097 Particles{Float64, 1} +run2 #6.62813 Particles{Float64, 1} +run3 #6.62813 Particles{Float64, 1} + +using Plots, StatsPlots +function lotka_volterra(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end + +# initial-value problem. +u0 = [1.0, 1.0] +p = [1.5, 1.0, 3.0, 1.0] +tspan = (0.0, 6.0) +prob = ODEProblem(lotka_volterra, u0, tspan, p) + +# Plot simulation. + +solution = solve(prob, Tsit5(); saveat = 0.05) +plot(solve(prob, Tsit5())) + +# Dataset creation for parameter estimation +time = solution.t +u = hcat(solution.u...) +x = u[1, :] + 0.5 * randn(length(u[1, :])) +y = u[2, :] + 0.5 * randn(length(u[1, :])) +dataset = [x, y, time] + +# Neural Networks must have 2 outputs as u -> [dx,dy] in function lotka_volterra() +chainflux = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) |> + Flux.f64 + +chainlux = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), Lux.Dense(6, 2)) + +alg1 = NeuralPDE.BNNODE(chainflux, + dataset = dataset, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol_flux_pestim = solve(prob, alg1) + +# Dataset not needed as we are solving the equation with ideal parameters +alg2 = NeuralPDE.BNNODE(chainlux, + draw_samples = 1000, + l2std = [ + 0.05, + 0.05, + ], + phystd = [ + 0.05, + 0.05, + ], + priorsNNw = (0.0, + 3.0), + n_leapfrog = 30, progress = true) + +sol_lux = solve(prob, alg2) + +#testing timepoints must match keyword arg `saveat`` timepoints of solve() call +t = collect(Float64, prob.tspan[1]:(1 / 50.0):prob.tspan[2]) + +# plotting solution for x,y for chain_flux +plot(t, sol_flux_pestim.ensemblesol[1]) +plot!(t, sol_flux_pestim.ensemblesol[2]) + +plot(sol_flux_pestim.ens1mblesol[1]) +plot!(sol_flux_pestim.ensemblesol[2]) + +# estimated ODE parameters by .estimated_ode_params, weights and biases by .estimated_nn_params +sol_flux_pestim.estimated_nn_params +sol_flux_pestim.estimated_ode_params + +# plotting solution for x,y for chain_lux +plot(t, sol_lux.ensemblesol[1]) +plot!(t, sol_lux.ensemblesol[2]) + +# estimated weights and biases by .estimated_nn_params for chain_lux +sol_lux.estimated_nn_params + +# # ----------------------------------stats----------------------------- +# # ---------------------------- +# # ----------------------------- +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:04:47 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:03:38 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:04:12 +# # -------------------------- +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:05:09 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:04:47 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:04:25 +# # -------------- +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:06:47 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:05:54 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:05:46 +# # ------------------------ +# # ----------------------- +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -882.2934218498742 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:04:06 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -882.2934218498742 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:03:32 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -882.2934218498742 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:03:01 +# # -------------------------- +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1411.1717435511828 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:04:02 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1411.1717435511828 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:04:08 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1411.1717435511828 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:04:15 +# # ---------------------------- +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -3240.067149411982 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:05:37 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -3240.067149411982 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:06:02 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -3240.067149411982 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:06:13 + +using NeuralPDE, Lux, ModelingToolkit, Optimization, OptimizationOptimJL +import ModelingToolkit: Interval, infimum, supremum + +using NeuralPDE, Flux, OptimizationOptimisers + +function diffeq(u, p, t) + u1, u2 = u + return [u2, p[1] + p[2] * sin(u1) + p[3] * u2] +end +p = [5, -10, -1.7] +u0 = [-1.0, 7.0] +tspan = (0.0, 10.0) +prob = ODEProblem(ODEFunction(diffeq), u0, tspan, p) + +chainnew = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), Flux.Dense(8, 2)) |> + Flux.f64 + +opt = OptimizationOptimisers.Adam(0.1) +opt = Optimisers.ADAGrad(0.1) +opt = Optimisers.AdaMax(0.01) +algnew = NeuralPDE.NNODE(chainnew, opt) +solution_new = solve(prob, algnew, verbose = true, + abstol = 1e-10, maxiters = 7000) +u = reduce(hcat, solution_new.u) +plot(solution_new.t, u[1, :]) +plot!(solution_new.t, u[2, :]) + +algnew = NeuralPDE.BNNODE(chainnew, draw_samples = 200, + n_leapfrog = 30, progress = true) +solution_new = solve(prob, algnew) + +@parameters t +@variables u1(..), u2(..) +D = Differential(t) +eq = [D(u1(t)) ~ u2(t), + D(u2(t)) ~ 5 - 10 * sin(u1(t)) - 1.7 * u2(t)]; + +import ModelingToolkit: Interval +bcs = [u1(0) ~ -1, u2(0) ~ 7] +domains = [t ∈ Interval(0.0, 10.0)] +dt = 0.01 + +input_ = length(domains) # number of dimensions +n = 16 +chain = [Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), Lux.Dense(n, 1)) + for _ in 1:2] + +@named pde_system = PDESystem(eq, bcs, domains, [t], [u1(t), u2(t)]) + +strategy = NeuralPDE.GridTraining(dt) +discretization = PhysicsInformedNN(chain, strategy) +sym_prob = NeuralPDE.symbolic_discretize(pde_system, discretization) + +pde_loss_functions = sym_prob.loss_functions.pde_loss_functions +bc_loss_functions = sym_prob.loss_functions.bc_loss_functions + +callback = function (p, l) + println("loss: ", l) + # println("pde_losses: ", map(l_ -> l_(p), pde_loss_functions)) + # println("bcs_losses: ", map(l_ -> l_(p), bc_loss_functions)) + return false +end + +loss_functions = [pde_loss_functions; bc_loss_functions] + +function loss_function(θ, p) + sum(map(l -> l(θ), loss_functions)) +end + +f_ = OptimizationFunction(loss_function, Optimization.AutoZygote()) +prob = Optimization.OptimizationProblem(f_, sym_prob.flat_init_params) + +res = Optimization.solve(prob, + OptimizationOptimJL.BFGS(); + callback = callback, + maxiters = 1000) +phi = discretization.phi \ No newline at end of file From b2f3ac1e4aeea8350005113fb8dd9f16bdc06421 Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Thu, 19 Oct 2023 17:48:04 -0400 Subject: [PATCH 02/53] Put new loglikelihood behind a conditional --- src/NeuralPDE.jl | 4 +- src/{ => bayesian}/BPINN_ode.jl | 6 +- src/{ => bayesian}/advancedHMC_MCMC.jl | 22 ++- src/bayesian/collocated_estim.jl | 194 +++++++++++++++++++++++++ test/bpinnexperimental.jl | 66 +++++++++ 5 files changed, 281 insertions(+), 11 deletions(-) rename src/{ => bayesian}/BPINN_ode.jl (98%) rename src/{ => bayesian}/advancedHMC_MCMC.jl (97%) create mode 100644 src/bayesian/collocated_estim.jl create mode 100644 test/bpinnexperimental.jl diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 945093ea04..e38fca98d4 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -50,8 +50,8 @@ include("rode_solve.jl") include("transform_inf_integral.jl") include("discretize.jl") include("neural_adapter.jl") -include("advancedHMC_MCMC.jl") -include("BPINN_ode.jl") +include("bayesian/advancedHMC_MCMC.jl") +include("bayesian/BPINN_ode.jl") export NNODE, TerminalPDEProblem, NNPDEHan, NNPDENS, NNRODE, KolmogorovPDEProblem, NNKolmogorov, NNStopping, ParamKolmogorovPDEProblem, diff --git a/src/BPINN_ode.jl b/src/bayesian/BPINN_ode.jl similarity index 98% rename from src/BPINN_ode.jl rename to src/bayesian/BPINN_ode.jl index da49640314..5c26329f14 100644 --- a/src/BPINN_ode.jl +++ b/src/bayesian/BPINN_ode.jl @@ -178,7 +178,8 @@ function DiffEqBase.__solve(prob::DiffEqBase.ODEProblem, verbose = false, saveat = 1 / 50.0, maxiters = nothing, - numensemble = floor(Int, alg.draw_samples / 3)) + numensemble = floor(Int, alg.draw_samples / 3), + estim_collocate = false) @unpack chain, l2std, phystd, param, priorsNNw, Kernel, strategy, draw_samples, dataset, init_params, nchains, physdt, Adaptorkwargs, Integratorkwargs, @@ -207,7 +208,8 @@ function DiffEqBase.__solve(prob::DiffEqBase.ODEProblem, Integratorkwargs = Integratorkwargs, MCMCkwargs = MCMCkwargs, progress = progress, - verbose = verbose) + verbose = verbose, + estim_collocate = estim_collocate) fullsolution = BPINNstats(mcmcchain, samples, statistics) ninv = length(param) diff --git a/src/advancedHMC_MCMC.jl b/src/bayesian/advancedHMC_MCMC.jl similarity index 97% rename from src/advancedHMC_MCMC.jl rename to src/bayesian/advancedHMC_MCMC.jl index 6032c7ca21..6b6b3303e7 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/bayesian/advancedHMC_MCMC.jl @@ -16,11 +16,12 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, physdt::Float64 extraparams::Int init_params::I + estim_collocate::Bool function LogTargetDensity(dim, prob, chain::Optimisers.Restructure, st, strategy, dataset, priors, phystd, l2std, autodiff, physdt, extraparams, - init_params::AbstractVector) + init_params::AbstractVector, estim_collocate) new{ typeof(chain), Nothing, @@ -39,12 +40,13 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, autodiff, physdt, extraparams, - init_params) + init_params, + estim_collocate) end function LogTargetDensity(dim, prob, chain::Lux.AbstractExplicitLayer, st, strategy, dataset, priors, phystd, l2std, autodiff, physdt, extraparams, - init_params::NamedTuple) + init_params::NamedTuple, estim_collocate) new{ typeof(chain), typeof(st), @@ -60,7 +62,8 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, autodiff, physdt, extraparams, - init_params) + init_params, + estim_collocate) end end @@ -79,7 +82,11 @@ function vector_to_parameters(ps_new::AbstractVector, ps::NamedTuple) end function LogDensityProblems.logdensity(Tar::LogTargetDensity, θ) - return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) + if Tar.estim_collocate + return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) + L2loss2(Tar, θ) + else + return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) + end end LogDensityProblems.dimension(Tar::LogTargetDensity) = Tar.dim @@ -481,7 +488,8 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), Integratorkwargs = (Integrator = Leapfrog,), MCMCkwargs = (n_leapfrog = 30,), - progress = false, verbose = false) + progress = false, verbose = false, + estim_collocate = false) # NN parameter prior mean and variance(PriorsNN must be a tuple) if isinplace(prob) @@ -542,7 +550,7 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; t0 = prob.tspan[1] # dimensions would be total no of params,initial_nnθ for Lux namedTuples ℓπ = LogTargetDensity(nparameters, prob, recon, st, strategy, dataset, priors, - phystd, l2std, autodiff, physdt, ninv, initial_nnθ) + phystd, l2std, autodiff, physdt, ninv, initial_nnθ, estim_collocate) try ℓπ(t0, initial_θ[1:(nparameters - ninv)]) diff --git a/src/bayesian/collocated_estim.jl b/src/bayesian/collocated_estim.jl new file mode 100644 index 0000000000..157388194e --- /dev/null +++ b/src/bayesian/collocated_estim.jl @@ -0,0 +1,194 @@ +# suggested extra loss function +function L2loss2(Tar::LogTargetDensity, θ) + f = Tar.prob.f + + # parameter estimation chosen or not + if Tar.extraparams > 0 + dataset, deri_sol = Tar.dataset + # deri_sol = deri_sol' + autodiff = Tar.autodiff + + # # Timepoints to enforce Physics + # dataset = Array(reduce(hcat, dataset)') + # t = dataset[end, :] + # û = dataset[1:(end - 1), :] + + # ode_params = Tar.extraparams == 1 ? + # θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : + # θ[((length(θ) - Tar.extraparams) + 1):length(θ)] + + # if length(û[:, 1]) == 1 + # physsol = [f(û[:, i][1], + # ode_params, + # t[i]) + # for i in 1:length(û[1, :])] + # else + # physsol = [f(û[:, i], + # ode_params, + # t[i]) + # for i in 1:length(û[1, :])] + # end + # #form of NN output matrix output dim x n + # deri_physsol = reduce(hcat, physsol) + + # > for perfect deriv(basically gradient matching in case of an ODEFunction) + # in case of PDE or general ODE we would want to reduce residue of f(du,u,p,t) + # if length(û[:, 1]) == 1 + # deri_sol = [f(û[:, i][1], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[1, :])] + # else + # deri_sol = [f(û[:, i], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[1, :])] + # end + # deri_sol = reduce(hcat, deri_sol) + # deri_sol = reduce(hcat, derivatives) + + # Timepoints to enforce Physics + t = dataset[end] + u1 = dataset[2] + û = dataset[1] + # Tar(t, θ[1:(length(θ) - Tar.extraparams)])' + # + + nnsol = NNodederi(Tar, t, θ[1:(length(θ) - Tar.extraparams)], autodiff) + + ode_params = Tar.extraparams == 1 ? + θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : + θ[((length(θ) - Tar.extraparams) + 1):length(θ)] + + if length(Tar.prob.u0) == 1 + physsol = [f(û[i], + ode_params, + t[i]) + for i in 1:length(û[:, 1])] + else + physsol = [f([û[i], u1[i]], + ode_params, + t[i]) + for i in 1:length(û[:, 1])] + end + #form of NN output matrix output dim x n + deri_physsol = reduce(hcat, physsol) + + # if length(Tar.prob.u0) == 1 + # nnsol = [f(û[i], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[:, 1])] + # else + # nnsol = [f([û[i], u1[i]], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[:, 1])] + # end + # form of NN output matrix output dim x n + # nnsol = reduce(hcat, nnsol) + + # > Instead of dataset gradients trying NN derivatives with dataset collocation + # # convert to matrix as nnsol + + physlogprob = 0 + for i in 1:length(Tar.prob.u0) + # can add phystd[i] for u[i] + physlogprob += logpdf(MvNormal(deri_physsol[i, :], + LinearAlgebra.Diagonal(map(abs2, + (Tar.l2std[i] * 4.0) .* + ones(length(nnsol[i, :]))))), + nnsol[i, :]) + end + return physlogprob + else + return 0 + end +end + +# PDE(DU,U,P,T)=0 + +# Derivated via Central Diff +# function calculate_derivatives2(dataset) +# x̂, time = dataset +# num_points = length(x̂) +# # Initialize an array to store the derivative values. +# derivatives = similar(x̂) + +# for i in 2:(num_points - 1) +# # Calculate the first-order derivative using central differences. +# Δt_forward = time[i + 1] - time[i] +# Δt_backward = time[i] - time[i - 1] + +# derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) + +# derivatives[i] = derivative +# end + +# # Derivatives at the endpoints can be calculated using forward or backward differences. +# derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) +# derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) +# return derivatives +# end + +function calderivatives(prob, dataset) + chainflux = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), + Flux.Dense(8, 2)) |> Flux.f64 + # chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 + function loss(x, y) + # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1]) + + # Flux.mse.(prob.u0[2] .+ (prob.tspan[2] .- x)' .* chainflux(x)[2, :], y[2])) + # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1])) + sum(Flux.mse.(chainflux(x), y)) + end + optimizer = Flux.Optimise.ADAM(0.01) + epochs = 3000 + for epoch in 1:epochs + Flux.train!(loss, + Flux.params(chainflux), + [(dataset[end]', dataset[1:(end - 1)])], + optimizer) + end + + # A1 = (prob.u0' .+ + # (prob.tspan[2] .- (dataset[end]' .+ sqrt(eps(eltype(Float64)))))' .* + # chainflux(dataset[end]' .+ sqrt(eps(eltype(Float64))))') + + # A2 = (prob.u0' .+ + # (prob.tspan[2] .- (dataset[end]'))' .* + # chainflux(dataset[end]')') + + A1 = chainflux(dataset[end]' .+ sqrt(eps(eltype(dataset[end][1])))) + A2 = chainflux(dataset[end]') + + gradients = (A2 .- A1) ./ sqrt(eps(eltype(dataset[end][1]))) + + return gradients +end + +function calculate_derivatives(dataset) + + # u = dataset[1] + # u1 = dataset[2] + # t = dataset[end] + # # control points + # n = Int(floor(length(t) / 10)) + # # spline for datasetvalues(solution) + # # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) + # interp = CubicSpline(u, t) + # interp1 = CubicSpline(u1, t) + # # derrivatives interpolation + # dx = t[2] - t[1] + # time = collect(t[1]:dx:t[end]) + # smoothu = [interp(i) for i in time] + # smoothu1 = [interp1(i) for i in time] + # # derivative of the spline (must match function derivative) + # û = tvdiff(smoothu, 20, 0.5, dx = dx, ε = 1) + # û1 = tvdiff(smoothu1, 20, 0.5, dx = dx, ε = 1) + # # tvdiff(smoothu, 100, 0.035, dx = dx, ε = 1) + # # FDM + # # û1 = diff(u) / dx + # # dataset[1] and smoothu are almost equal(rounding errors) + # return [û, û1] + +end \ No newline at end of file diff --git a/test/bpinnexperimental.jl b/test/bpinnexperimental.jl new file mode 100644 index 0000000000..153124b069 --- /dev/null +++ b/test/bpinnexperimental.jl @@ -0,0 +1,66 @@ +using Test, MCMCChains +using ForwardDiff, Distributions, OrdinaryDiffEq +using Flux, OptimizationOptimisers, AdvancedHMC, Lux +using Statistics, Random, Functors, ComponentArrays +using NeuralPDE, MonteCarloMeasurements + +Random.seed!(110) + +using NeuralPDE, Lux, Plots, OrdinaryDiffEq, Distributions, Random + +function lotka_volterra(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end + +# initial-value problem. +u0 = [1.0, 1.0] +p = [1.5, 1.0, 3.0, 1.0] +tspan = (0.0, 4.0) +prob = ODEProblem(lotka_volterra, u0, tspan, p) + +# Solve using OrdinaryDiffEq.jl solver +dt = 0.01 +solution = solve(prob, Tsit5(); saveat = dt) + +times = solution.t +u = hcat(solution.u...) +x = u[1, :] + (u[1, :]) .* (0.05 .* randn(length(u[1, :]))) +y = u[2, :] + (u[2, :]) .* (0.05 .* randn(length(u[2, :]))) +dataset = [x, y, times] + +plot(times, x, label = "noisy x") +plot!(times, y, label = "noisy y") +plot!(solution, labels = ["x" "y"]) + +chain = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), + Lux.Dense(6, 2)) + +alg = BNNODE(chain; +dataset = dataset, +draw_samples = 1000, +l2std = [0.1, 0.1], +phystd = [0.1, 0.1], +priorsNNw = (0.0, 3.0), +param = [ + Normal(1, 2), + Normal(2, 2), + Normal(2, 2), + Normal(0, 2)], progress = false) + +sol_pestim = solve(prob, alg; saveat = dt) +plot(times, sol_pestim.ensemblesol[1], label = "estimated x") +plot!(times, sol_pestim.ensemblesol[2], label = "estimated y") + +# comparing it with the original solution +plot!(solution, labels = ["true x" "true y"]) + +sol_pestim.estimated_ode_params \ No newline at end of file From 058aa05eeb5dc434c825c30115b8ea7fd2d733ca Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Fri, 27 Oct 2023 16:58:42 -0400 Subject: [PATCH 03/53] fitzhughnagumo experiment and some edits --- src/NeuralPDE.jl | 1 + src/bayesian/advancedHMC_MCMC.jl | 6 +-- src/bayesian/collocated_estim.jl | 10 ++--- test/bpinnexperimental.jl | 68 ++++++++++++++++++++++++++++---- 4 files changed, 68 insertions(+), 17 deletions(-) diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index e38fca98d4..edfaf9664a 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -52,6 +52,7 @@ include("discretize.jl") include("neural_adapter.jl") include("bayesian/advancedHMC_MCMC.jl") include("bayesian/BPINN_ode.jl") +include("bayesian/collocated_estim.jl") export NNODE, TerminalPDEProblem, NNPDEHan, NNPDENS, NNRODE, KolmogorovPDEProblem, NNKolmogorov, NNStopping, ParamKolmogorovPDEProblem, diff --git a/src/bayesian/advancedHMC_MCMC.jl b/src/bayesian/advancedHMC_MCMC.jl index 6b6b3303e7..740bb344a3 100644 --- a/src/bayesian/advancedHMC_MCMC.jl +++ b/src/bayesian/advancedHMC_MCMC.jl @@ -587,8 +587,8 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; MCMC_alg = kernelchoice(Kernel, MCMCkwargs) Kernel = AdvancedHMC.make_kernel(MCMC_alg, integrator) - samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, adaptor; - progress = progress, verbose = verbose) + samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, adaptor, draw_samples; + progress = progress, verbose = verbose, drop_warmup = true) samplesc[i] = samples statsc[i] = stats @@ -606,7 +606,7 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; MCMC_alg = kernelchoice(Kernel, MCMCkwargs) Kernel = AdvancedHMC.make_kernel(MCMC_alg, integrator) samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, - adaptor; progress = progress, verbose = verbose) + adaptor, draw_samples; progress = progress, verbose = verbose, drop_warmup = true) # return a chain(basic chain),samples and stats matrix_samples = hcat(samples...) diff --git a/src/bayesian/collocated_estim.jl b/src/bayesian/collocated_estim.jl index 157388194e..b113b76f12 100644 --- a/src/bayesian/collocated_estim.jl +++ b/src/bayesian/collocated_estim.jl @@ -4,10 +4,8 @@ function L2loss2(Tar::LogTargetDensity, θ) # parameter estimation chosen or not if Tar.extraparams > 0 - dataset, deri_sol = Tar.dataset # deri_sol = deri_sol' autodiff = Tar.autodiff - # # Timepoints to enforce Physics # dataset = Array(reduce(hcat, dataset)') # t = dataset[end, :] @@ -48,9 +46,9 @@ function L2loss2(Tar::LogTargetDensity, θ) # deri_sol = reduce(hcat, derivatives) # Timepoints to enforce Physics - t = dataset[end] - u1 = dataset[2] - û = dataset[1] + t = Tar.dataset[end] + u1 = Tar.dataset[2] + û = Tar.dataset[1] # Tar(t, θ[1:(length(θ) - Tar.extraparams)])' # @@ -69,7 +67,7 @@ function L2loss2(Tar::LogTargetDensity, θ) physsol = [f([û[i], u1[i]], ode_params, t[i]) - for i in 1:length(û[:, 1])] + for i in 1:length(û)] end #form of NN output matrix output dim x n deri_physsol = reduce(hcat, physsol) diff --git a/test/bpinnexperimental.jl b/test/bpinnexperimental.jl index 153124b069..ffe7fcf0f8 100644 --- a/test/bpinnexperimental.jl +++ b/test/bpinnexperimental.jl @@ -28,13 +28,13 @@ tspan = (0.0, 4.0) prob = ODEProblem(lotka_volterra, u0, tspan, p) # Solve using OrdinaryDiffEq.jl solver -dt = 0.01 +dt = 0.2 solution = solve(prob, Tsit5(); saveat = dt) times = solution.t u = hcat(solution.u...) -x = u[1, :] + (u[1, :]) .* (0.05 .* randn(length(u[1, :]))) -y = u[2, :] + (u[2, :]) .* (0.05 .* randn(length(u[2, :]))) +x = u[1, :] + (u[1, :]) .* (0.3 .* randn(length(u[1, :]))) +y = u[2, :] + (u[2, :]) .* (0.3 .* randn(length(u[2, :]))) dataset = [x, y, times] plot(times, x, label = "noisy x") @@ -54,13 +54,65 @@ param = [ Normal(1, 2), Normal(2, 2), Normal(2, 2), - Normal(0, 2)], progress = false) + Normal(0, 2)], progress = true) -sol_pestim = solve(prob, alg; saveat = dt) -plot(times, sol_pestim.ensemblesol[1], label = "estimated x") -plot!(times, sol_pestim.ensemblesol[2], label = "estimated y") +@time sol_pestim1 = solve(prob, alg; saveat = dt,) +@time sol_pestim2 = solve(prob, alg; estim_collocate = true, saveat = dt) +plot(times, sol_pestim1.ensemblesol[1], label = "estimated x1") +plot!(times, sol_pestim2.ensemblesol[1], label = "estimated x2") +plot!(times, sol_pestim1.ensemblesol[2], label = "estimated y1") +plot!(times, sol_pestim2.ensemblesol[2], label = "estimated y2") # comparing it with the original solution plot!(solution, labels = ["true x" "true y"]) -sol_pestim.estimated_ode_params \ No newline at end of file +@show sol_pestim1.estimated_ode_params +@show sol_pestim2.estimated_ode_params + +function fitz(u, p , t) + v, w = u[1], u[2] + a,b,τinv,l = p[1], p[2], p[3], p[4] + + dv = v - 0.33*v^3 -w + l + dw = τinv*(v + a - b*w) + + return [dv, dw] +end + +prob_ode_fitzhughnagumo = ODEProblem(fitz, [1.0,1.0], (0.0,10.0), [0.7,0.8,1/12.5,0.5]) +dt = 0.5 +sol = solve(prob_ode_fitzhughnagumo, Tsit5(), saveat = dt) + +sig = 0.20 +data = Array(sol) +dataset = [data[1,:] .+ (sig .* rand(length(sol.t))), data[2, :] .+ (sig .* rand(length(sol.t))), sol.t] +priors = [truncated(Normal(0.5,1.0),0,1.5), truncated(Normal(0.5,1.0),0,1.5), truncated(Normal(0.0,0.5),0.0,0.5), truncated(Normal(0.5,1.0),0,1)] + + +plot(sol.t, dataset[1], label = "noisy x") +plot!(sol.t, dataset[2], label = "noisy y") +plot!(sol, labels = ["x" "y"]) + +chain = Lux.Chain(Lux.Dense(1, 10, tanh), Lux.Dense(10, 10, tanh), + Lux.Dense(10, 2)) + +Adaptorkwargs = (Adaptor = AdvancedHMC.StanHMCAdaptor, + Metric = AdvancedHMC.DiagEuclideanMetric, targetacceptancerate = 0.65) +alg = BNNODE(chain; +dataset = dataset, +draw_samples = 10000, +l2std = [0.1, 0.1], +phystd = [0.1, 0.1], +priorsNNw = (0.01, 3.0), +Adaptorkwargs = Adaptorkwargs, +param = priors, progress = true) + +@time sol_pestim1 = solve(prob_ode_fitzhughnagumo, alg; saveat = dt) +@time sol_pestim2 = solve(prob_ode_fitzhughnagumo, alg; estim_collocate = true, saveat = dt) +plot!(sol.t, sol_pestim1.ensemblesol[1], label = "estimated x1") +plot!(sol.t, sol_pestim2.ensemblesol[1], label = "estimated x2") +plot!(sol.t, sol_pestim1.ensemblesol[2], label = "estimated y1") +plot!(sol.t, sol_pestim2.ensemblesol[2], label = "estimated y2") + +@show sol_pestim1.estimated_ode_params +@show sol_pestim2.estimated_ode_params \ No newline at end of file From 103e1febf7f0d4153b560ad8a962106c6bf92cde Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Sat, 28 Oct 2023 15:14:31 -0400 Subject: [PATCH 04/53] Scale logpdfs and fix chain creation --- src/bayesian/BPINN_ode.jl | 6 +++++- src/bayesian/advancedHMC_MCMC.jl | 13 ++++++------- test/bpinnexperimental.jl | 22 +++++++++++----------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/bayesian/BPINN_ode.jl b/src/bayesian/BPINN_ode.jl index 5c26329f14..a2cce9db34 100644 --- a/src/bayesian/BPINN_ode.jl +++ b/src/bayesian/BPINN_ode.jl @@ -217,8 +217,12 @@ function DiffEqBase.__solve(prob::DiffEqBase.ODEProblem, if chain isa Lux.AbstractExplicitLayer θinit, st = Lux.setup(Random.default_rng(), chain) + println(length(θinit)) + println(length(samples[1])) + println(draw_samples) θ = [vector_to_parameters(samples[i][1:(end - ninv)], θinit) - for i in (draw_samples - numensemble):draw_samples] + for i in 1:max(draw_samples - draw_samples ÷ 10, draw_samples - 1000)] + luxar = [chain(t', θ[i], st)[1] for i in 1:numensemble] # only need for size θinit = collect(ComponentArrays.ComponentArray(θinit)) diff --git a/src/bayesian/advancedHMC_MCMC.jl b/src/bayesian/advancedHMC_MCMC.jl index 740bb344a3..5e995ebfdb 100644 --- a/src/bayesian/advancedHMC_MCMC.jl +++ b/src/bayesian/advancedHMC_MCMC.jl @@ -83,9 +83,9 @@ end function LogDensityProblems.logdensity(Tar::LogTargetDensity, θ) if Tar.estim_collocate - return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) + L2loss2(Tar, θ) + return physloglikelihood(Tar, θ)/length(Tar.dataset[1]) + priorweights(Tar, θ) + L2LossData(Tar, θ)/length(Tar.dataset[1]) + L2loss2(Tar, θ)/length(Tar.dataset[1]) else - return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) + return physloglikelihood(Tar, θ)/length(Tar.dataset[1]) + priorweights(Tar, θ) + L2LossData(Tar, θ)/length(Tar.dataset[1]) end end @@ -587,7 +587,7 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; MCMC_alg = kernelchoice(Kernel, MCMCkwargs) Kernel = AdvancedHMC.make_kernel(MCMC_alg, integrator) - samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, adaptor, draw_samples; + samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, adaptor; progress = progress, verbose = verbose, drop_warmup = true) samplesc[i] = samples @@ -606,11 +606,10 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; MCMC_alg = kernelchoice(Kernel, MCMCkwargs) Kernel = AdvancedHMC.make_kernel(MCMC_alg, integrator) samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, - adaptor, draw_samples; progress = progress, verbose = verbose, drop_warmup = true) - + adaptor; progress = progress, verbose = verbose, drop_warmup = true) # return a chain(basic chain),samples and stats - matrix_samples = hcat(samples...) - mcmc_chain = MCMCChains.Chains(matrix_samples') + matrix_samples = reshape(hcat(samples...), (length(samples[1]), length(samples), 1)) + mcmc_chain = MCMCChains.Chains(matrix_samples) return mcmc_chain, samples, stats end end \ No newline at end of file diff --git a/test/bpinnexperimental.jl b/test/bpinnexperimental.jl index ffe7fcf0f8..3de049bf58 100644 --- a/test/bpinnexperimental.jl +++ b/test/bpinnexperimental.jl @@ -86,7 +86,7 @@ sol = solve(prob_ode_fitzhughnagumo, Tsit5(), saveat = dt) sig = 0.20 data = Array(sol) dataset = [data[1,:] .+ (sig .* rand(length(sol.t))), data[2, :] .+ (sig .* rand(length(sol.t))), sol.t] -priors = [truncated(Normal(0.5,1.0),0,1.5), truncated(Normal(0.5,1.0),0,1.5), truncated(Normal(0.0,0.5),0.0,0.5), truncated(Normal(0.5,1.0),0,1)] +priors = [Normal(0.5,1.0), Normal(0.5,1.0), Normal(0.0,0.5), Normal(0.5,1.0)] plot(sol.t, dataset[1], label = "noisy x") @@ -97,22 +97,22 @@ chain = Lux.Chain(Lux.Dense(1, 10, tanh), Lux.Dense(10, 10, tanh), Lux.Dense(10, 2)) Adaptorkwargs = (Adaptor = AdvancedHMC.StanHMCAdaptor, - Metric = AdvancedHMC.DiagEuclideanMetric, targetacceptancerate = 0.65) + Metric = AdvancedHMC.DiagEuclideanMetric, targetacceptancerate = 0.8) alg = BNNODE(chain; dataset = dataset, -draw_samples = 10000, +draw_samples = 1000, l2std = [0.1, 0.1], phystd = [0.1, 0.1], priorsNNw = (0.01, 3.0), Adaptorkwargs = Adaptorkwargs, param = priors, progress = true) -@time sol_pestim1 = solve(prob_ode_fitzhughnagumo, alg; saveat = dt) -@time sol_pestim2 = solve(prob_ode_fitzhughnagumo, alg; estim_collocate = true, saveat = dt) -plot!(sol.t, sol_pestim1.ensemblesol[1], label = "estimated x1") -plot!(sol.t, sol_pestim2.ensemblesol[1], label = "estimated x2") -plot!(sol.t, sol_pestim1.ensemblesol[2], label = "estimated y1") -plot!(sol.t, sol_pestim2.ensemblesol[2], label = "estimated y2") +@time sol_pestim3 = solve(prob_ode_fitzhughnagumo, alg; saveat = dt) +@time sol_pestim4 = solve(prob_ode_fitzhughnagumo, alg; estim_collocate = true, saveat = dt) +plot!(sol.t, sol_pestim3.ensemblesol[1], label = "estimated x1") +plot!(sol.t, sol_pestim4.ensemblesol[1], label = "estimated x2") +plot!(sol.t, sol_pestim3.ensemblesol[2], label = "estimated y1") +plot!(sol.t, sol_pestim4.ensemblesol[2], label = "estimated y2") -@show sol_pestim1.estimated_ode_params -@show sol_pestim2.estimated_ode_params \ No newline at end of file +@show sol_pestim3.estimated_ode_params +@show sol_pestim4.estimated_ode_params From f5b4f1cb7e97a06a1b56b39efcfba33734fc74f3 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sat, 20 Jan 2024 23:50:27 +0530 Subject: [PATCH 05/53] trying to sync --- src/BNNODE_new.jl | 794 +++++++ src/BPINN_ode.jl | 1 - src/advancedHMC_MCMC.jl | 392 +--- test/BPINN_Tests.jl | 4084 +----------------------------------- test/BPINN_newform.jl | 4354 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 5287 insertions(+), 4338 deletions(-) create mode 100644 src/BNNODE_new.jl create mode 100644 test/BPINN_newform.jl diff --git a/src/BNNODE_new.jl b/src/BNNODE_new.jl new file mode 100644 index 0000000000..e6b1f24faa --- /dev/null +++ b/src/BNNODE_new.jl @@ -0,0 +1,794 @@ +mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, + P <: Vector{<:Distribution}, + D <: + Union{Vector{Nothing}, Vector{<:Vector{<:AbstractFloat}}}, +} + dim::Int + prob::DiffEqBase.ODEProblem + chain::C + st::S + strategy::ST + dataset::D + priors::P + phystd::Vector{Float64} + l2std::Vector{Float64} + autodiff::Bool + physdt::Float64 + extraparams::Int + init_params::I + + function LogTargetDensity(dim, prob, chain::Optimisers.Restructure, st, strategy, + dataset, + priors, phystd, l2std, autodiff, physdt, extraparams, + init_params::AbstractVector) + new{ + typeof(chain), + Nothing, + typeof(strategy), + typeof(init_params), + typeof(priors), + typeof(dataset), + }(dim, + prob, + chain, + nothing, strategy, + dataset, + priors, + phystd, + l2std, + autodiff, + physdt, + extraparams, + init_params) + end + function LogTargetDensity(dim, prob, chain::Lux.AbstractExplicitLayer, st, strategy, + dataset, + priors, phystd, l2std, autodiff, physdt, extraparams, + init_params::NamedTuple) + new{ + typeof(chain), + typeof(st), + typeof(strategy), + typeof(init_params), + typeof(priors), + typeof(dataset), + }(dim, + prob, + chain, st, strategy, + dataset, priors, + phystd, l2std, + autodiff, + physdt, + extraparams, + init_params) + end +end + +""" +cool function to convert parameter's vector to ComponentArray of parameters (for Lux Chain: vector of samples -> Lux ComponentArrays) +""" +function vector_to_parameters(ps_new::AbstractVector, ps::NamedTuple) + @assert length(ps_new) == Lux.parameterlength(ps) + i = 1 + function get_ps(x) + z = reshape(view(ps_new, i:(i + length(x) - 1)), size(x)) + i += length(x) + return z + end + return Functors.fmap(get_ps, ps) +end + +function LogDensityProblems.logdensity(Tar::LogTargetDensity, θ) + return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) + # + L2loss2(Tar, θ) +end + +LogDensityProblems.dimension(Tar::LogTargetDensity) = Tar.dim + +function LogDensityProblems.capabilities(::LogTargetDensity) + LogDensityProblems.LogDensityOrder{1}() +end + +# suggested extra loss function +function L2loss2(Tar::LogTargetDensity, θ) + f = Tar.prob.f + + # parameter estimation chosen or not + if Tar.extraparams > 0 + dataset, deri_sol = Tar.dataset + # deri_sol = deri_sol' + autodiff = Tar.autodiff + + # # Timepoints to enforce Physics + # dataset = Array(reduce(hcat, dataset)') + # t = dataset[end, :] + # û = dataset[1:(end - 1), :] + + # ode_params = Tar.extraparams == 1 ? + # θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : + # θ[((length(θ) - Tar.extraparams) + 1):length(θ)] + + # if length(û[:, 1]) == 1 + # physsol = [f(û[:, i][1], + # ode_params, + # t[i]) + # for i in 1:length(û[1, :])] + # else + # physsol = [f(û[:, i], + # ode_params, + # t[i]) + # for i in 1:length(û[1, :])] + # end + # #form of NN output matrix output dim x n + # deri_physsol = reduce(hcat, physsol) + + # > for perfect deriv(basically gradient matching in case of an ODEFunction) + # in case of PDE or general ODE we would want to reduce residue of f(du,u,p,t) + # if length(û[:, 1]) == 1 + # deri_sol = [f(û[:, i][1], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[1, :])] + # else + # deri_sol = [f(û[:, i], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[1, :])] + # end + # deri_sol = reduce(hcat, deri_sol) + # deri_sol = reduce(hcat, derivatives) + + # Timepoints to enforce Physics + t = dataset[end] + u1 = dataset[2] + û = dataset[1] + # Tar(t, θ[1:(length(θ) - Tar.extraparams)])' + # + + nnsol = NNodederi(Tar, t, θ[1:(length(θ) - Tar.extraparams)], autodiff) + + ode_params = Tar.extraparams == 1 ? + θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : + θ[((length(θ) - Tar.extraparams) + 1):length(θ)] + + if length(Tar.prob.u0) == 1 + physsol = [f(û[i], + ode_params, + t[i]) + for i in 1:length(û[:, 1])] + else + physsol = [f([û[i], u1[i]], + ode_params, + t[i]) + for i in 1:length(û[:, 1])] + end + #form of NN output matrix output dim x n + deri_physsol = reduce(hcat, physsol) + + # if length(Tar.prob.u0) == 1 + # nnsol = [f(û[i], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[:, 1])] + # else + # nnsol = [f([û[i], u1[i]], + # Tar.prob.p, + # t[i]) + # for i in 1:length(û[:, 1])] + # end + # form of NN output matrix output dim x n + # nnsol = reduce(hcat, nnsol) + + # > Instead of dataset gradients trying NN derivatives with dataset collocation + # # convert to matrix as nnsol + + physlogprob = 0 + for i in 1:length(Tar.prob.u0) + # can add phystd[i] for u[i] + physlogprob += logpdf(MvNormal(deri_physsol[i, :], + LinearAlgebra.Diagonal(map(abs2, + (Tar.l2std[i] * 4.0) .* + ones(length(nnsol[i, :]))))), + nnsol[i, :]) + end + return physlogprob + else + return 0 + end +end + +# PDE(DU,U,P,T)=0 + +# Derivated via Central Diff +# function calculate_derivatives2(dataset) +# x̂, time = dataset +# num_points = length(x̂) +# # Initialize an array to store the derivative values. +# derivatives = similar(x̂) + +# for i in 2:(num_points - 1) +# # Calculate the first-order derivative using central differences. +# Δt_forward = time[i + 1] - time[i] +# Δt_backward = time[i] - time[i - 1] + +# derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) + +# derivatives[i] = derivative +# end + +# # Derivatives at the endpoints can be calculated using forward or backward differences. +# derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) +# derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) +# return derivatives +# end + +function calderivatives(prob, dataset) + chainflux = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), + Flux.Dense(8, 2)) |> Flux.f64 + # chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 + function loss(x, y) + # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1]) + + # Flux.mse.(prob.u0[2] .+ (prob.tspan[2] .- x)' .* chainflux(x)[2, :], y[2])) + # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1])) + sum(Flux.mse.(chainflux(x), y)) + end + optimizer = Flux.Optimise.ADAM(0.01) + epochs = 3000 + for epoch in 1:epochs + Flux.train!(loss, + Flux.params(chainflux), + [(dataset[end]', dataset[1:(end - 1)])], + optimizer) + end + + # A1 = (prob.u0' .+ + # (prob.tspan[2] .- (dataset[end]' .+ sqrt(eps(eltype(Float64)))))' .* + # chainflux(dataset[end]' .+ sqrt(eps(eltype(Float64))))') + + # A2 = (prob.u0' .+ + # (prob.tspan[2] .- (dataset[end]'))' .* + # chainflux(dataset[end]')') + + A1 = chainflux(dataset[end]' .+ sqrt(eps(eltype(dataset[end][1])))) + A2 = chainflux(dataset[end]') + + gradients = (A2 .- A1) ./ sqrt(eps(eltype(dataset[end][1]))) + + return gradients +end + +function calculate_derivatives(dataset) + + # u = dataset[1] + # u1 = dataset[2] + # t = dataset[end] + # # control points + # n = Int(floor(length(t) / 10)) + # # spline for datasetvalues(solution) + # # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) + # interp = CubicSpline(u, t) + # interp1 = CubicSpline(u1, t) + # # derrivatives interpolation + # dx = t[2] - t[1] + # time = collect(t[1]:dx:t[end]) + # smoothu = [interp(i) for i in time] + # smoothu1 = [interp1(i) for i in time] + # # derivative of the spline (must match function derivative) + # û = tvdiff(smoothu, 20, 0.5, dx = dx, ε = 1) + # û1 = tvdiff(smoothu1, 20, 0.5, dx = dx, ε = 1) + # # tvdiff(smoothu, 100, 0.035, dx = dx, ε = 1) + # # FDM + # # û1 = diff(u) / dx + # # dataset[1] and smoothu are almost equal(rounding errors) + # return [û, û1] + +end + +""" +L2 loss loglikelihood(needed for ODE parameter estimation) +""" +function L2LossData(Tar::LogTargetDensity, θ) + dataset = Tar.dataset + # check if dataset is provided + if dataset isa Vector{Nothing} || Tar.extraparams == 0 + return 0 + else + # matrix(each row corresponds to vector u's rows) + nn = Tar(dataset[end], θ[1:(length(θ) - Tar.extraparams)]) + + L2logprob = 0 + for i in 1:length(Tar.prob.u0) + # for u[i] ith vector must be added to dataset,nn[1,:] is the dx in lotka_volterra + L2logprob += logpdf(MvNormal(nn[i, :], + LinearAlgebra.Diagonal(map(abs2, + (Tar.l2std[i] * 0.5) .* + ones(length(dataset[i]))))), + dataset[i]) + end + return L2logprob + end +end + +""" +physics loglikelihood over problem timespan + dataset timepoints +""" +function physloglikelihood(Tar::LogTargetDensity, θ) + f = Tar.prob.f + p = Tar.prob.p + tspan = Tar.prob.tspan + autodiff = Tar.autodiff + strategy = Tar.strategy + + # parameter estimation chosen or not + if Tar.extraparams > 0 + ode_params = Tar.extraparams == 1 ? + θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : + θ[((length(θ) - Tar.extraparams) + 1):length(θ)] + else + ode_params = p == SciMLBase.NullParameters() ? [] : p + end + + return getlogpdf(strategy, Tar, f, autodiff, tspan, ode_params, θ) +end + +function getlogpdf(strategy::GridTraining, Tar::LogTargetDensity, f, autodiff::Bool, + tspan, + ode_params, θ) + if Tar.dataset isa Vector{Nothing} + t = collect(eltype(strategy.dx), tspan[1]:(strategy.dx):tspan[2]) + else + t = vcat(collect(eltype(strategy.dx), tspan[1]:(strategy.dx):tspan[2]), + Tar.dataset[end]) + end + + sum(innerdiff(Tar, f, autodiff, t, θ, + ode_params)) +end + +function getlogpdf(strategy::StochasticTraining, + Tar::LogTargetDensity, + f, + autodiff::Bool, + tspan, + ode_params, + θ) + if Tar.dataset isa Vector{Nothing} + t = [(tspan[2] - tspan[1]) * rand() + tspan[1] for i in 1:(strategy.points)] + else + t = vcat([(tspan[2] - tspan[1]) * rand() + tspan[1] for i in 1:(strategy.points)], + Tar.dataset[end]) + end + + sum(innerdiff(Tar, f, autodiff, t, θ, + ode_params)) +end + +function getlogpdf(strategy::QuadratureTraining, Tar::LogTargetDensity, f, + autodiff::Bool, + tspan, + ode_params, θ) + function integrand(t::Number, θ) + innerdiff(Tar, f, autodiff, [t], θ, ode_params) + end + intprob = IntegralProblem(integrand, tspan[1], tspan[2], θ; nout = length(Tar.prob.u0)) + # add dataset logpdf? + sol = solve(intprob, QuadGKJL(); abstol = strategy.abstol, reltol = strategy.reltol) + sum(sol.u) +end + +function getlogpdf(strategy::WeightedIntervalTraining, Tar::LogTargetDensity, f, + autodiff::Bool, + tspan, + ode_params, θ) + minT = tspan[1] + maxT = tspan[2] + + weights = strategy.weights ./ sum(strategy.weights) + + N = length(weights) + points = strategy.points + + difference = (maxT - minT) / N + + data = Float64[] + for (index, item) in enumerate(weights) + temp_data = rand(1, trunc(Int, points * item)) .* difference .+ minT .+ + ((index - 1) * difference) + data = append!(data, temp_data) + end + + if Tar.dataset isa Vector{Nothing} + t = data + else + t = vcat(data, + Tar.dataset[end]) + end + + sum(innerdiff(Tar, f, autodiff, t, θ, + ode_params)) +end + +""" +MvNormal likelihood at each `ti` in time `t` for ODE collocation residue with NN with parameters θ +""" +function innerdiff(Tar::LogTargetDensity, f, autodiff::Bool, t::AbstractVector, θ, + ode_params) + + # Tar used for phi and LogTargetDensity object attributes access + out = Tar(t, θ[1:(length(θ) - Tar.extraparams)]) + + # # reject samples case(write clear reason why) + if any(isinf, out[:, 1]) || any(isinf, ode_params) + return -Inf + end + + # this is a vector{vector{dx,dy}}(handle case single u(float passed)) + if length(out[:, 1]) == 1 + physsol = [f(out[:, i][1], + ode_params, + t[i]) + for i in 1:length(out[1, :])] + else + physsol = [f(out[:, i], + ode_params, + t[i]) + for i in 1:length(out[1, :])] + end + physsol = reduce(hcat, physsol) + + nnsol = NNodederi(Tar, t, θ[1:(length(θ) - Tar.extraparams)], autodiff) + + vals = nnsol .- physsol + + # N dimensional vector if N outputs for NN(each row has logpdf of i[i] where u is vector of dependant variables) + return [logpdf(MvNormal(vals[i, :], + LinearAlgebra.Diagonal(map(abs2, + Tar.phystd[i] .* + ones(length(vals[i, :]))))), + zeros(length(vals[i, :]))) for i in 1:length(Tar.prob.u0)] +end + +""" +prior logpdf for NN parameters + ODE constants +""" +function priorweights(Tar::LogTargetDensity, θ) + allparams = Tar.priors + # nn weights + nnwparams = allparams[1] + + if Tar.extraparams > 0 + # Vector of ode parameters priors + invpriors = allparams[2:end] + + invlogpdf = sum(logpdf(invpriors[length(θ) - i + 1], θ[i]) + for i in (length(θ) - Tar.extraparams + 1):length(θ); init = 0.0) + + return (invlogpdf + + + logpdf(nnwparams, θ[1:(length(θ) - Tar.extraparams)])) + else + return logpdf(nnwparams, θ) + end +end + +function generate_Tar(chain::Lux.AbstractExplicitLayer, init_params) + θ, st = Lux.setup(Random.default_rng(), chain) + return init_params, chain, st +end + +function generate_Tar(chain::Lux.AbstractExplicitLayer, init_params::Nothing) + θ, st = Lux.setup(Random.default_rng(), chain) + return θ, chain, st +end + +function generate_Tar(chain::Flux.Chain, init_params) + θ, re = Flux.destructure(chain) + return init_params, re, nothing +end + +function generate_Tar(chain::Flux.Chain, init_params::Nothing) + θ, re = Flux.destructure(chain) + # find_good_stepsize,phasepoint takes only float64 + return θ, re, nothing +end + +""" +nn OUTPUT AT t,θ ~ phi(t,θ) +""" +function (f::LogTargetDensity{C, S})(t::AbstractVector, + θ) where {C <: Optimisers.Restructure, S} + f.prob.u0 .+ (t' .- f.prob.tspan[1]) .* f.chain(θ)(adapt(parameterless_type(θ), t')) +end + +function (f::LogTargetDensity{C, S})(t::AbstractVector, + θ) where {C <: Lux.AbstractExplicitLayer, S} + θ = vector_to_parameters(θ, f.init_params) + y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), t'), θ, f.st) + ChainRulesCore.@ignore_derivatives f.st = st + f.prob.u0 .+ (t' .- f.prob.tspan[1]) .* y +end + +function (f::LogTargetDensity{C, S})(t::Number, + θ) where {C <: Optimisers.Restructure, S} + # must handle paired odes hence u0 broadcasted + f.prob.u0 .+ (t - f.prob.tspan[1]) * f.chain(θ)(adapt(parameterless_type(θ), [t])) +end + +function (f::LogTargetDensity{C, S})(t::Number, + θ) where {C <: Lux.AbstractExplicitLayer, S} + θ = vector_to_parameters(θ, f.init_params) + y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), [t]), θ, f.st) + ChainRulesCore.@ignore_derivatives f.st = st + f.prob.u0 .+ (t .- f.prob.tspan[1]) .* y +end + +""" +similar to ode_dfdx() in NNODE/ode_solve.jl +""" +function NNodederi(phi::LogTargetDensity, t::AbstractVector, θ, autodiff::Bool) + if autodiff + hcat(ForwardDiff.derivative.(ti -> phi(ti, θ), t)...) + else + (phi(t .+ sqrt(eps(eltype(t))), θ) - phi(t, θ)) ./ sqrt(eps(eltype(t))) + end +end + +function kernelchoice(Kernel, max_depth, Δ_max, n_leapfrog, δ, λ) + if Kernel == HMC + Kernel(n_leapfrog) + elseif Kernel == HMCDA + Kernel(δ, λ) + else + Kernel(δ, max_depth = max_depth, Δ_max = Δ_max) + end +end + +function integratorchoice(Integrator, initial_ϵ, jitter_rate, + tempering_rate) + if Integrator == JitteredLeapfrog + Integrator(initial_ϵ, jitter_rate) + elseif Integrator == TemperedLeapfrog + Integrator(initial_ϵ, tempering_rate) + else + Integrator(initial_ϵ) + end +end + +function adaptorchoice(Adaptor, mma, ssa) + if Adaptor != AdvancedHMC.NoAdaptation() + Adaptor(mma, ssa) + else + AdvancedHMC.NoAdaptation() + end +end + +""" +```julia +ahmc_bayesian_pinn_ode(prob, chain; strategy = GridTraining, + dataset = [nothing],init_params = nothing, + draw_samples = 1000, physdt = 1 / 20.0f0,l2std = [0.05], + phystd = [0.05], priorsNNw = (0.0, 2.0), + param = [],nchains = 1,autodiff = false, Kernel = HMC, + Integrator = Leapfrog, Adaptor = StanHMCAdaptor, + targetacceptancerate = 0.8, Metric = DiagEuclideanMetric, + jitter_rate = 3.0, tempering_rate = 3.0, max_depth = 10, + Δ_max = 1000, n_leapfrog = 10, δ = 0.65, λ = 0.3, + progress = false,verbose = false) +``` +!!! warn + + Note that ahmc_bayesian_pinn_ode() only supports ODEs which are written in the out-of-place form, i.e. + `du = f(u,p,t)`, and not `f(du,u,p,t)`. If not declared out-of-place, then the ahmc_bayesian_pinn_ode() + will exit with an error. + +## Example +linear = (u, p, t) -> -u / p[1] + exp(t / p[2]) * cos(t) +tspan = (0.0, 10.0) +u0 = 0.0 +p = [5.0, -5.0] +prob = ODEProblem(linear, u0, tspan, p) + +# CREATE DATASET (Necessity for accurate Parameter estimation) +sol = solve(prob, Tsit5(); saveat = 0.05) +u = sol.u[1:100] +time = sol.t[1:100] + +# dataset and BPINN create +x̂ = collect(Float64, Array(u) + 0.05 * randn(size(u))) +dataset = [x̂, time] + +chainflux1 = Flux.Chain(Flux.Dense(1, 5, tanh), Flux.Dense(5, 5, tanh), Flux.Dense(5, 1) + +# simply solving ode here hence better to not pass dataset(uses ode params specified in prob) +fh_mcmc_chainflux1, fhsamplesflux1, fhstatsflux1 = ahmc_bayesian_pinn_ode(prob,chainflux1, + dataset = dataset, + draw_samples = 1500, + l2std = [0.05], + phystd = [0.05], + priorsNNw = (0.0,3.0)) + +# solving ode + estimating parameters hence dataset needed to optimize parameters upon + Pior Distributions for ODE params +fh_mcmc_chainflux2, fhsamplesflux2, fhstatsflux2 = ahmc_bayesian_pinn_ode(prob,chainflux1, + dataset = dataset, + draw_samples = 1500, + l2std = [0.05], + phystd = [0.05], + priorsNNw = (0.0,3.0), + param = [Normal(6.5,0.5),Normal(-3,0.5)]) + +## NOTES +Dataset is required for accurate Parameter estimation + solving equations +Incase you are only solving the Equations for solution, do not provide dataset + +## Positional Arguments +* `prob`: DEProblem(out of place and the function signature should be f(u,p,t) +* `chain`: Lux/Flux Neural Netork which would be made the Bayesian PINN + +## Keyword Arguments +* `strategy`: The training strategy used to choose the points for the evaluations. By default GridTraining is used with given physdt discretization. +* `dataset`: Vector containing Vectors of corresponding u,t values +* `init_params`: intial parameter values for BPINN (ideally for multiple chains different initializations preferred) +* `nchains`: number of chains you want to sample (random initialisation of params by default) +* `draw_samples`: number of samples to be drawn in the MCMC algorithms (warmup samples are ~2/3 of draw samples) +* `l2std`: standard deviation of BPINN predicition against L2 losses/Dataset +* `phystd`: standard deviation of BPINN predicition against Chosen Underlying ODE System +* `priorsNNw`: Vector of [mean, std] for BPINN parameter. Weights and Biases of BPINN are Normal Distributions by default +* `param`: Vector of chosen ODE parameters Distributions in case of Inverse problems. +* `autodiff`: Boolean Value for choice of Derivative Backend(default is numerical) +* `physdt`: Timestep for approximating ODE in it's Time domain. (1/20.0 by default) + +# AHMC.jl is still developing convenience structs so might need changes on new releases. +* `Kernel`: Choice of MCMC Sampling Algorithm (AdvancedHMC.jl implemenations HMC/NUTS/HMCDA) +* `targetacceptancerate`: Target percentage(in decimal) of iterations in which the proposals were accepted(0.8 by default) +* `Integrator(jitter_rate, tempering_rate), Metric, Adaptor`: https://turinglang.org/AdvancedHMC.jl/stable/ +* `max_depth`: Maximum doubling tree depth (NUTS) +* `Δ_max`: Maximum divergence during doubling tree (NUTS) +* `n_leapfrog`: number of leapfrog steps for HMC +* `δ`: target acceptance probability for NUTS/HMCDA +* `λ`: target trajectory length for HMCDA +* `progress`: controls whether to show the progress meter or not. +* `verbose`: controls the verbosity. (Sample call args in AHMC) + +""" + +""" +dataset would be (x̂,t) +priors: pdf for W,b + pdf for ODE params +""" +function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; + strategy = GridTraining, dataset = [nothing], + init_params = nothing, draw_samples = 1000, + physdt = 1 / 20.0, l2std = [0.05], + phystd = [0.05], priorsNNw = (0.0, 2.0), + param = [], nchains = 1, autodiff = false, + Kernel = HMC, Integrator = Leapfrog, + Adaptor = StanHMCAdaptor, targetacceptancerate = 0.8, + Metric = DiagEuclideanMetric, jitter_rate = 3.0, + tempering_rate = 3.0, max_depth = 10, Δ_max = 1000, + n_leapfrog = 10, δ = 0.65, λ = 0.3, progress = false, + verbose = false) + + # NN parameter prior mean and variance(PriorsNN must be a tuple) + if isinplace(prob) + throw(error("The BPINN ODE solver only supports out-of-place ODE definitions, i.e. du=f(u,p,t).")) + end + + strategy = strategy == GridTraining ? strategy(physdt) : strategy + + if dataset != [nothing] && + (length(dataset) < 2 || !(typeof(dataset) <: Vector{<:Vector{<:AbstractFloat}})) + throw(error("Invalid dataset. dataset would be timeseries (x̂,t) where type: Vector{Vector{AbstractFloat}")) + end + + if dataset != [nothing] && param == [] + println("Dataset is only needed for Parameter Estimation + Forward Problem, not in only Forward Problem case.") + elseif dataset == [nothing] && param != [] + throw(error("Dataset Required for Parameter Estimation.")) + end + + if chain isa Lux.AbstractExplicitLayer || chain isa Flux.Chain + # Flux-vector, Lux-Named Tuple + initial_nnθ, recon, st = generate_Tar(chain, init_params) + else + error("Only Lux.AbstractExplicitLayer and Flux.Chain neural networks are supported") + end + + if nchains > Threads.nthreads() + throw(error("number of chains is greater than available threads")) + elseif nchains < 1 + throw(error("number of chains must be greater than 1")) + end + + # eltype(physdt) cause needs Float64 for find_good_stepsize + if chain isa Lux.AbstractExplicitLayer + # Lux chain(using component array later as vector_to_parameter need namedtuple) + initial_θ = collect(eltype(physdt), + vcat(ComponentArrays.ComponentArray(initial_nnθ))) + else + initial_θ = collect(eltype(physdt), initial_nnθ) + end + + # adding ode parameter estimation + nparameters = length(initial_θ) + ninv = length(param) + priors = [ + MvNormal(priorsNNw[1] * ones(nparameters), + LinearAlgebra.Diagonal(map(abs2, priorsNNw[2] .* ones(nparameters)))), + ] + + # append Ode params to all paramvector + if ninv > 0 + # shift ode params(initialise ode params by prior means) + initial_θ = vcat(initial_θ, [Distributions.params(param[i])[1] for i in 1:ninv]) + priors = vcat(priors, param) + nparameters += ninv + end + + t0 = prob.tspan[1] + # dimensions would be total no of params,initial_nnθ for Lux namedTuples + ℓπ = LogTargetDensity(nparameters, prob, recon, st, strategy, dataset, priors, + phystd, l2std, autodiff, physdt, ninv, initial_nnθ) + + try + ℓπ(t0, initial_θ[1:(nparameters - ninv)]) + catch err + if isa(err, DimensionMismatch) + throw(DimensionMismatch("Dimensions of the initial u0 and chain should match")) + else + throw(err) + end + end + + # Define Hamiltonian system (nparameters ~ dimensionality of the sampling space) + metric = Metric(nparameters) + hamiltonian = Hamiltonian(metric, ℓπ, ForwardDiff) + + println("physics Logpdf is : ", physloglikelihood(ℓπ, initial_θ)) + println("prior Logpdf is : ", priorweights(ℓπ, initial_θ)) + println("L2lossData Logpdf is : ", L2LossData(ℓπ, initial_θ)) + println("L2loss2 Logpdf is : ", L2loss2(ℓπ, initial_θ)) + + # parallel sampling option + if nchains != 1 + # Cache to store the chains + chains = Vector{Any}(undef, nchains) + statsc = Vector{Any}(undef, nchains) + samplesc = Vector{Any}(undef, nchains) + + Threads.@threads for i in 1:nchains + # each chain has different initial NNparameter values(better posterior exploration) + initial_θ = vcat(randn(nparameters - ninv), + initial_θ[(nparameters - ninv + 1):end]) + initial_ϵ = find_good_stepsize(hamiltonian, initial_θ) + integrator = integratorchoice(Integrator, initial_ϵ, jitter_rate, + tempering_rate) + adaptor = adaptorchoice(Adaptor, MassMatrixAdaptor(metric), + StepSizeAdaptor(targetacceptancerate, integrator)) + Kernel = AdvancedHMC.make_kernel(kernelchoice(Kernel, max_depth, Δ_max, + n_leapfrog, δ, λ), integrator) + samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, adaptor; + progress = progress, verbose = verbose) + + samplesc[i] = samples + statsc[i] = stats + mcmc_chain = Chains(hcat(samples...)') + chains[i] = mcmc_chain + end + + return chains, samplesc, statsc + else + initial_ϵ = find_good_stepsize(hamiltonian, initial_θ) + integrator = integratorchoice(Integrator, initial_ϵ, jitter_rate, tempering_rate) + adaptor = adaptorchoice(Adaptor, MassMatrixAdaptor(metric), + StepSizeAdaptor(targetacceptancerate, integrator)) + Kernel = AdvancedHMC.make_kernel(kernelchoice(Kernel, max_depth, Δ_max, n_leapfrog, + δ, λ), integrator) + samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, + adaptor; progress = progress, verbose = verbose) + + # return a chain(basic chain),samples and stats + matrix_samples = hcat(samples...) + mcmc_chain = MCMCChains.Chains(matrix_samples') + return mcmc_chain, samples, stats + end +end \ No newline at end of file diff --git a/src/BPINN_ode.jl b/src/BPINN_ode.jl index f9a68b8917..f79f5208f2 100644 --- a/src/BPINN_ode.jl +++ b/src/BPINN_ode.jl @@ -284,7 +284,6 @@ function DiffEqBase.__solve(prob::DiffEqBase.ODEProblem, push!(ensemblecurves, ensemblecurve) end - # estimated using all samples nnparams = length(θinit) estimnnparams = [Particles(reduce(hcat, samples)[i, :]) for i in 1:nnparams] diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index e6b1f24faa..6fee4a818e 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -18,9 +18,9 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, init_params::I function LogTargetDensity(dim, prob, chain::Optimisers.Restructure, st, strategy, - dataset, - priors, phystd, l2std, autodiff, physdt, extraparams, - init_params::AbstractVector) + dataset, + priors, phystd, l2std, autodiff, physdt, extraparams, + init_params::AbstractVector) new{ typeof(chain), Nothing, @@ -42,9 +42,9 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, init_params) end function LogTargetDensity(dim, prob, chain::Lux.AbstractExplicitLayer, st, strategy, - dataset, - priors, phystd, l2std, autodiff, physdt, extraparams, - init_params::NamedTuple) + dataset, + priors, phystd, l2std, autodiff, physdt, extraparams, + init_params::NamedTuple) new{ typeof(chain), typeof(st), @@ -65,9 +65,11 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, end """ -cool function to convert parameter's vector to ComponentArray of parameters (for Lux Chain: vector of samples -> Lux ComponentArrays) +function needed for converting vector of sampled parameters into ComponentVector in case of Lux chain output, derivatives +the sampled parameters are of exotic type `Dual` due to ForwardDiff's autodiff tagging """ -function vector_to_parameters(ps_new::AbstractVector, ps::NamedTuple) +function vector_to_parameters(ps_new::AbstractVector, + ps::Union{NamedTuple, ComponentArrays.ComponentVector}) @assert length(ps_new) == Lux.parameterlength(ps) i = 1 function get_ps(x) @@ -78,9 +80,10 @@ function vector_to_parameters(ps_new::AbstractVector, ps::NamedTuple) return Functors.fmap(get_ps, ps) end +vector_to_parameters(ps_new::AbstractVector, ps::AbstractVector) = ps_new + function LogDensityProblems.logdensity(Tar::LogTargetDensity, θ) return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) - # + L2loss2(Tar, θ) end LogDensityProblems.dimension(Tar::LogTargetDensity) = Tar.dim @@ -89,221 +92,24 @@ function LogDensityProblems.capabilities(::LogTargetDensity) LogDensityProblems.LogDensityOrder{1}() end -# suggested extra loss function -function L2loss2(Tar::LogTargetDensity, θ) - f = Tar.prob.f - - # parameter estimation chosen or not - if Tar.extraparams > 0 - dataset, deri_sol = Tar.dataset - # deri_sol = deri_sol' - autodiff = Tar.autodiff - - # # Timepoints to enforce Physics - # dataset = Array(reduce(hcat, dataset)') - # t = dataset[end, :] - # û = dataset[1:(end - 1), :] - - # ode_params = Tar.extraparams == 1 ? - # θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : - # θ[((length(θ) - Tar.extraparams) + 1):length(θ)] - - # if length(û[:, 1]) == 1 - # physsol = [f(û[:, i][1], - # ode_params, - # t[i]) - # for i in 1:length(û[1, :])] - # else - # physsol = [f(û[:, i], - # ode_params, - # t[i]) - # for i in 1:length(û[1, :])] - # end - # #form of NN output matrix output dim x n - # deri_physsol = reduce(hcat, physsol) - - # > for perfect deriv(basically gradient matching in case of an ODEFunction) - # in case of PDE or general ODE we would want to reduce residue of f(du,u,p,t) - # if length(û[:, 1]) == 1 - # deri_sol = [f(û[:, i][1], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[1, :])] - # else - # deri_sol = [f(û[:, i], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[1, :])] - # end - # deri_sol = reduce(hcat, deri_sol) - # deri_sol = reduce(hcat, derivatives) - - # Timepoints to enforce Physics - t = dataset[end] - u1 = dataset[2] - û = dataset[1] - # Tar(t, θ[1:(length(θ) - Tar.extraparams)])' - # - - nnsol = NNodederi(Tar, t, θ[1:(length(θ) - Tar.extraparams)], autodiff) - - ode_params = Tar.extraparams == 1 ? - θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : - θ[((length(θ) - Tar.extraparams) + 1):length(θ)] - - if length(Tar.prob.u0) == 1 - physsol = [f(û[i], - ode_params, - t[i]) - for i in 1:length(û[:, 1])] - else - physsol = [f([û[i], u1[i]], - ode_params, - t[i]) - for i in 1:length(û[:, 1])] - end - #form of NN output matrix output dim x n - deri_physsol = reduce(hcat, physsol) - - # if length(Tar.prob.u0) == 1 - # nnsol = [f(û[i], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[:, 1])] - # else - # nnsol = [f([û[i], u1[i]], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[:, 1])] - # end - # form of NN output matrix output dim x n - # nnsol = reduce(hcat, nnsol) - - # > Instead of dataset gradients trying NN derivatives with dataset collocation - # # convert to matrix as nnsol - - physlogprob = 0 - for i in 1:length(Tar.prob.u0) - # can add phystd[i] for u[i] - physlogprob += logpdf(MvNormal(deri_physsol[i, :], - LinearAlgebra.Diagonal(map(abs2, - (Tar.l2std[i] * 4.0) .* - ones(length(nnsol[i, :]))))), - nnsol[i, :]) - end - return physlogprob - else - return 0 - end -end - -# PDE(DU,U,P,T)=0 - -# Derivated via Central Diff -# function calculate_derivatives2(dataset) -# x̂, time = dataset -# num_points = length(x̂) -# # Initialize an array to store the derivative values. -# derivatives = similar(x̂) - -# for i in 2:(num_points - 1) -# # Calculate the first-order derivative using central differences. -# Δt_forward = time[i + 1] - time[i] -# Δt_backward = time[i] - time[i - 1] - -# derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) - -# derivatives[i] = derivative -# end - -# # Derivatives at the endpoints can be calculated using forward or backward differences. -# derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) -# derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) -# return derivatives -# end - -function calderivatives(prob, dataset) - chainflux = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), - Flux.Dense(8, 2)) |> Flux.f64 - # chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 - function loss(x, y) - # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1]) + - # Flux.mse.(prob.u0[2] .+ (prob.tspan[2] .- x)' .* chainflux(x)[2, :], y[2])) - # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1])) - sum(Flux.mse.(chainflux(x), y)) - end - optimizer = Flux.Optimise.ADAM(0.01) - epochs = 3000 - for epoch in 1:epochs - Flux.train!(loss, - Flux.params(chainflux), - [(dataset[end]', dataset[1:(end - 1)])], - optimizer) - end - - # A1 = (prob.u0' .+ - # (prob.tspan[2] .- (dataset[end]' .+ sqrt(eps(eltype(Float64)))))' .* - # chainflux(dataset[end]' .+ sqrt(eps(eltype(Float64))))') - - # A2 = (prob.u0' .+ - # (prob.tspan[2] .- (dataset[end]'))' .* - # chainflux(dataset[end]')') - - A1 = chainflux(dataset[end]' .+ sqrt(eps(eltype(dataset[end][1])))) - A2 = chainflux(dataset[end]') - - gradients = (A2 .- A1) ./ sqrt(eps(eltype(dataset[end][1]))) - - return gradients -end - -function calculate_derivatives(dataset) - - # u = dataset[1] - # u1 = dataset[2] - # t = dataset[end] - # # control points - # n = Int(floor(length(t) / 10)) - # # spline for datasetvalues(solution) - # # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) - # interp = CubicSpline(u, t) - # interp1 = CubicSpline(u1, t) - # # derrivatives interpolation - # dx = t[2] - t[1] - # time = collect(t[1]:dx:t[end]) - # smoothu = [interp(i) for i in time] - # smoothu1 = [interp1(i) for i in time] - # # derivative of the spline (must match function derivative) - # û = tvdiff(smoothu, 20, 0.5, dx = dx, ε = 1) - # û1 = tvdiff(smoothu1, 20, 0.5, dx = dx, ε = 1) - # # tvdiff(smoothu, 100, 0.035, dx = dx, ε = 1) - # # FDM - # # û1 = diff(u) / dx - # # dataset[1] and smoothu are almost equal(rounding errors) - # return [û, û1] - -end - """ L2 loss loglikelihood(needed for ODE parameter estimation) """ function L2LossData(Tar::LogTargetDensity, θ) - dataset = Tar.dataset # check if dataset is provided - if dataset isa Vector{Nothing} || Tar.extraparams == 0 + if Tar.dataset isa Vector{Nothing} || Tar.extraparams == 0 return 0 else # matrix(each row corresponds to vector u's rows) - nn = Tar(dataset[end], θ[1:(length(θ) - Tar.extraparams)]) + nn = Tar(Tar.dataset[end], θ[1:(length(θ) - Tar.extraparams)]) L2logprob = 0 for i in 1:length(Tar.prob.u0) # for u[i] ith vector must be added to dataset,nn[1,:] is the dx in lotka_volterra L2logprob += logpdf(MvNormal(nn[i, :], - LinearAlgebra.Diagonal(map(abs2, - (Tar.l2std[i] * 0.5) .* - ones(length(dataset[i]))))), - dataset[i]) + LinearAlgebra.Diagonal(abs2.(Tar.l2std[i] .* + ones(length(Tar.dataset[i]))))), + Tar.dataset[i]) end return L2logprob end @@ -332,8 +138,8 @@ function physloglikelihood(Tar::LogTargetDensity, θ) end function getlogpdf(strategy::GridTraining, Tar::LogTargetDensity, f, autodiff::Bool, - tspan, - ode_params, θ) + tspan, + ode_params, θ) if Tar.dataset isa Vector{Nothing} t = collect(eltype(strategy.dx), tspan[1]:(strategy.dx):tspan[2]) else @@ -346,12 +152,12 @@ function getlogpdf(strategy::GridTraining, Tar::LogTargetDensity, f, autodiff::B end function getlogpdf(strategy::StochasticTraining, - Tar::LogTargetDensity, - f, - autodiff::Bool, - tspan, - ode_params, - θ) + Tar::LogTargetDensity, + f, + autodiff::Bool, + tspan, + ode_params, + θ) if Tar.dataset isa Vector{Nothing} t = [(tspan[2] - tspan[1]) * rand() + tspan[1] for i in 1:(strategy.points)] else @@ -364,22 +170,21 @@ function getlogpdf(strategy::StochasticTraining, end function getlogpdf(strategy::QuadratureTraining, Tar::LogTargetDensity, f, - autodiff::Bool, - tspan, - ode_params, θ) + autodiff::Bool, + tspan, + ode_params, θ) function integrand(t::Number, θ) innerdiff(Tar, f, autodiff, [t], θ, ode_params) end intprob = IntegralProblem(integrand, tspan[1], tspan[2], θ; nout = length(Tar.prob.u0)) - # add dataset logpdf? sol = solve(intprob, QuadGKJL(); abstol = strategy.abstol, reltol = strategy.reltol) sum(sol.u) end function getlogpdf(strategy::WeightedIntervalTraining, Tar::LogTargetDensity, f, - autodiff::Bool, - tspan, - ode_params, θ) + autodiff::Bool, + tspan, + ode_params, θ) minT = tspan[1] maxT = tspan[2] @@ -412,7 +217,7 @@ end MvNormal likelihood at each `ti` in time `t` for ODE collocation residue with NN with parameters θ """ function innerdiff(Tar::LogTargetDensity, f, autodiff::Bool, t::AbstractVector, θ, - ode_params) + ode_params) # Tar used for phi and LogTargetDensity object attributes access out = Tar(t, θ[1:(length(θ) - Tar.extraparams)]) @@ -442,9 +247,8 @@ function innerdiff(Tar::LogTargetDensity, f, autodiff::Bool, t::AbstractVector, # N dimensional vector if N outputs for NN(each row has logpdf of i[i] where u is vector of dependant variables) return [logpdf(MvNormal(vals[i, :], - LinearAlgebra.Diagonal(map(abs2, - Tar.phystd[i] .* - ones(length(vals[i, :]))))), + LinearAlgebra.Diagonal(abs2.(Tar.phystd[i] .* + ones(length(vals[i, :]))))), zeros(length(vals[i, :]))) for i in 1:length(Tar.prob.u0)] end @@ -496,12 +300,12 @@ end nn OUTPUT AT t,θ ~ phi(t,θ) """ function (f::LogTargetDensity{C, S})(t::AbstractVector, - θ) where {C <: Optimisers.Restructure, S} + θ) where {C <: Optimisers.Restructure, S} f.prob.u0 .+ (t' .- f.prob.tspan[1]) .* f.chain(θ)(adapt(parameterless_type(θ), t')) end function (f::LogTargetDensity{C, S})(t::AbstractVector, - θ) where {C <: Lux.AbstractExplicitLayer, S} + θ) where {C <: Lux.AbstractExplicitLayer, S} θ = vector_to_parameters(θ, f.init_params) y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), t'), θ, f.st) ChainRulesCore.@ignore_derivatives f.st = st @@ -509,13 +313,13 @@ function (f::LogTargetDensity{C, S})(t::AbstractVector, end function (f::LogTargetDensity{C, S})(t::Number, - θ) where {C <: Optimisers.Restructure, S} + θ) where {C <: Optimisers.Restructure, S} # must handle paired odes hence u0 broadcasted f.prob.u0 .+ (t - f.prob.tspan[1]) * f.chain(θ)(adapt(parameterless_type(θ), [t])) end function (f::LogTargetDensity{C, S})(t::Number, - θ) where {C <: Lux.AbstractExplicitLayer, S} + θ) where {C <: Lux.AbstractExplicitLayer, S} θ = vector_to_parameters(θ, f.init_params) y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), [t]), θ, f.st) ChainRulesCore.@ignore_derivatives f.st = st @@ -533,21 +337,27 @@ function NNodederi(phi::LogTargetDensity, t::AbstractVector, θ, autodiff::Bool) end end -function kernelchoice(Kernel, max_depth, Δ_max, n_leapfrog, δ, λ) - if Kernel == HMC - Kernel(n_leapfrog) - elseif Kernel == HMCDA +function kernelchoice(Kernel, MCMCkwargs) + if Kernel == HMCDA + δ, λ = MCMCkwargs[:δ], MCMCkwargs[:λ] Kernel(δ, λ) - else + elseif Kernel == NUTS + δ, max_depth, Δ_max = MCMCkwargs[:δ], MCMCkwargs[:max_depth], MCMCkwargs[:Δ_max] Kernel(δ, max_depth = max_depth, Δ_max = Δ_max) + else + # HMC + n_leapfrog = MCMCkwargs[:n_leapfrog] + Kernel(n_leapfrog) end end -function integratorchoice(Integrator, initial_ϵ, jitter_rate, - tempering_rate) +function integratorchoice(Integratorkwargs, initial_ϵ) + Integrator = Integratorkwargs[:Integrator] if Integrator == JitteredLeapfrog + jitter_rate = Integratorkwargs[:jitter_rate] Integrator(initial_ϵ, jitter_rate) elseif Integrator == TemperedLeapfrog + tempering_rate = Integratorkwargs[:tempering_rate] Integrator(initial_ϵ, tempering_rate) else Integrator(initial_ϵ) @@ -568,12 +378,12 @@ ahmc_bayesian_pinn_ode(prob, chain; strategy = GridTraining, dataset = [nothing],init_params = nothing, draw_samples = 1000, physdt = 1 / 20.0f0,l2std = [0.05], phystd = [0.05], priorsNNw = (0.0, 2.0), - param = [],nchains = 1,autodiff = false, Kernel = HMC, - Integrator = Leapfrog, Adaptor = StanHMCAdaptor, - targetacceptancerate = 0.8, Metric = DiagEuclideanMetric, - jitter_rate = 3.0, tempering_rate = 3.0, max_depth = 10, - Δ_max = 1000, n_leapfrog = 10, δ = 0.65, λ = 0.3, - progress = false,verbose = false) + param = [], nchains = 1, autodiff = false, Kernel = HMC, + Adaptorkwargs = (Adaptor = StanHMCAdaptor, + Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), + Integratorkwargs = (Integrator = Leapfrog,), + MCMCkwargs = (n_leapfrog = 30,), + progress = false, verbose = false) ``` !!! warn @@ -626,47 +436,48 @@ Incase you are only solving the Equations for solution, do not provide dataset ## Keyword Arguments * `strategy`: The training strategy used to choose the points for the evaluations. By default GridTraining is used with given physdt discretization. -* `dataset`: Vector containing Vectors of corresponding u,t values * `init_params`: intial parameter values for BPINN (ideally for multiple chains different initializations preferred) -* `nchains`: number of chains you want to sample (random initialisation of params by default) +* `nchains`: number of chains you want to sample * `draw_samples`: number of samples to be drawn in the MCMC algorithms (warmup samples are ~2/3 of draw samples) -* `l2std`: standard deviation of BPINN predicition against L2 losses/Dataset -* `phystd`: standard deviation of BPINN predicition against Chosen Underlying ODE System -* `priorsNNw`: Vector of [mean, std] for BPINN parameter. Weights and Biases of BPINN are Normal Distributions by default +* `l2std`: standard deviation of BPINN prediction against L2 losses/Dataset +* `phystd`: standard deviation of BPINN prediction against Chosen Underlying ODE System +* `priorsNNw`: Tuple of (mean, std) for BPINN Network parameters. Weights and Biases of BPINN are Normal Distributions by default. * `param`: Vector of chosen ODE parameters Distributions in case of Inverse problems. * `autodiff`: Boolean Value for choice of Derivative Backend(default is numerical) * `physdt`: Timestep for approximating ODE in it's Time domain. (1/20.0 by default) -# AHMC.jl is still developing convenience structs so might need changes on new releases. +# AdvancedHMC.jl is still developing convenience structs so might need changes on new releases. * `Kernel`: Choice of MCMC Sampling Algorithm (AdvancedHMC.jl implemenations HMC/NUTS/HMCDA) -* `targetacceptancerate`: Target percentage(in decimal) of iterations in which the proposals were accepted(0.8 by default) -* `Integrator(jitter_rate, tempering_rate), Metric, Adaptor`: https://turinglang.org/AdvancedHMC.jl/stable/ -* `max_depth`: Maximum doubling tree depth (NUTS) -* `Δ_max`: Maximum divergence during doubling tree (NUTS) -* `n_leapfrog`: number of leapfrog steps for HMC -* `δ`: target acceptance probability for NUTS/HMCDA -* `λ`: target trajectory length for HMCDA +* `Integratorkwargs`: `Integrator`, `jitter_rate`, `tempering_rate`. Refer: https://turinglang.org/AdvancedHMC.jl/stable/ +* `Adaptorkwargs`: `Adaptor`, `Metric`, `targetacceptancerate`. Refer: https://turinglang.org/AdvancedHMC.jl/stable/ + Note: Target percentage(in decimal) of iterations in which the proposals are accepted (0.8 by default) +* `MCMCargs`: A NamedTuple containing all the chosen MCMC kernel's(HMC/NUTS/HMCDA) Arguments, as follows : + * `n_leapfrog`: number of leapfrog steps for HMC + * `δ`: target acceptance probability for NUTS and HMCDA + * `λ`: target trajectory length for HMCDA + * `max_depth`: Maximum doubling tree depth (NUTS) + * `Δ_max`: Maximum divergence during doubling tree (NUTS) + Refer: https://turinglang.org/AdvancedHMC.jl/stable/ * `progress`: controls whether to show the progress meter or not. * `verbose`: controls the verbosity. (Sample call args in AHMC) """ """ -dataset would be (x̂,t) priors: pdf for W,b + pdf for ODE params """ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; - strategy = GridTraining, dataset = [nothing], - init_params = nothing, draw_samples = 1000, - physdt = 1 / 20.0, l2std = [0.05], - phystd = [0.05], priorsNNw = (0.0, 2.0), - param = [], nchains = 1, autodiff = false, - Kernel = HMC, Integrator = Leapfrog, - Adaptor = StanHMCAdaptor, targetacceptancerate = 0.8, - Metric = DiagEuclideanMetric, jitter_rate = 3.0, - tempering_rate = 3.0, max_depth = 10, Δ_max = 1000, - n_leapfrog = 10, δ = 0.65, λ = 0.3, progress = false, - verbose = false) + strategy = GridTraining, dataset = [nothing], + init_params = nothing, draw_samples = 1000, + physdt = 1 / 20.0, l2std = [0.05], + phystd = [0.05], priorsNNw = (0.0, 2.0), + param = [], nchains = 1, autodiff = false, + Kernel = HMC, + Adaptorkwargs = (Adaptor = StanHMCAdaptor, + Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), + Integratorkwargs = (Integrator = Leapfrog,), + MCMCkwargs = (n_leapfrog = 30,), + progress = false, verbose = false) # NN parameter prior mean and variance(PriorsNN must be a tuple) if isinplace(prob) @@ -676,7 +487,7 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; strategy = strategy == GridTraining ? strategy(physdt) : strategy if dataset != [nothing] && - (length(dataset) < 2 || !(typeof(dataset) <: Vector{<:Vector{<:AbstractFloat}})) + (length(dataset) < 2 || !(dataset isa Vector{<:Vector{<:AbstractFloat}})) throw(error("Invalid dataset. dataset would be timeseries (x̂,t) where type: Vector{Vector{AbstractFloat}")) end @@ -713,7 +524,7 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; ninv = length(param) priors = [ MvNormal(priorsNNw[1] * ones(nparameters), - LinearAlgebra.Diagonal(map(abs2, priorsNNw[2] .* ones(nparameters)))), + LinearAlgebra.Diagonal(abs2.(priorsNNw[2] .* ones(nparameters)))), ] # append Ode params to all paramvector @@ -739,15 +550,17 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; end end + @info("Current Physics Log-likelihood : ", physloglikelihood(ℓπ, initial_θ)) + @info("Current Prior Log-likelihood : ", priorweights(ℓπ, initial_θ)) + @info("Current MSE against dataset Log-likelihood : ", L2LossData(ℓπ, initial_θ)) + + Adaptor, Metric, targetacceptancerate = Adaptorkwargs[:Adaptor], + Adaptorkwargs[:Metric], Adaptorkwargs[:targetacceptancerate] + # Define Hamiltonian system (nparameters ~ dimensionality of the sampling space) metric = Metric(nparameters) hamiltonian = Hamiltonian(metric, ℓπ, ForwardDiff) - println("physics Logpdf is : ", physloglikelihood(ℓπ, initial_θ)) - println("prior Logpdf is : ", priorweights(ℓπ, initial_θ)) - println("L2lossData Logpdf is : ", L2LossData(ℓπ, initial_θ)) - println("L2loss2 Logpdf is : ", L2loss2(ℓπ, initial_θ)) - # parallel sampling option if nchains != 1 # Cache to store the chains @@ -760,12 +573,12 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; initial_θ = vcat(randn(nparameters - ninv), initial_θ[(nparameters - ninv + 1):end]) initial_ϵ = find_good_stepsize(hamiltonian, initial_θ) - integrator = integratorchoice(Integrator, initial_ϵ, jitter_rate, - tempering_rate) + integrator = integratorchoice(Integratorkwargs, initial_ϵ) adaptor = adaptorchoice(Adaptor, MassMatrixAdaptor(metric), StepSizeAdaptor(targetacceptancerate, integrator)) - Kernel = AdvancedHMC.make_kernel(kernelchoice(Kernel, max_depth, Δ_max, - n_leapfrog, δ, λ), integrator) + + MCMC_alg = kernelchoice(Kernel, MCMCkwargs) + Kernel = AdvancedHMC.make_kernel(MCMC_alg, integrator) samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, adaptor; progress = progress, verbose = verbose) @@ -778,14 +591,21 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; return chains, samplesc, statsc else initial_ϵ = find_good_stepsize(hamiltonian, initial_θ) - integrator = integratorchoice(Integrator, initial_ϵ, jitter_rate, tempering_rate) + integrator = integratorchoice(Integratorkwargs, initial_ϵ) adaptor = adaptorchoice(Adaptor, MassMatrixAdaptor(metric), StepSizeAdaptor(targetacceptancerate, integrator)) - Kernel = AdvancedHMC.make_kernel(kernelchoice(Kernel, max_depth, Δ_max, n_leapfrog, - δ, λ), integrator) + + MCMC_alg = kernelchoice(Kernel, MCMCkwargs) + Kernel = AdvancedHMC.make_kernel(MCMC_alg, integrator) samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, adaptor; progress = progress, verbose = verbose) + @info("Sampling Complete.") + @info("Current Physics Log-likelihood : ", physloglikelihood(ℓπ, samples[end])) + @info("Current Prior Log-likelihood : ", priorweights(ℓπ, samples[end])) + @info("Current MSE against dataset Log-likelihood : ", + L2LossData(ℓπ, samples[end])) + # return a chain(basic chain),samples and stats matrix_samples = hcat(samples...) mcmc_chain = MCMCChains.Chains(matrix_samples') diff --git a/test/BPINN_Tests.jl b/test/BPINN_Tests.jl index fa2f04073e..cb0303daf0 100644 --- a/test/BPINN_Tests.jl +++ b/test/BPINN_Tests.jl @@ -9,18 +9,6 @@ using NeuralPDE, MonteCarloMeasurements # on latest Julia version it performs much better for below tests Random.seed!(100) -# for sampled params->lux ComponentArray -function vector_to_parameters(ps_new::AbstractVector, ps::NamedTuple) - @assert length(ps_new) == Lux.parameterlength(ps) - i = 1 - function get_ps(x) - z = reshape(view(ps_new, i:(i + length(x) - 1)), size(x)) - i += length(x) - return z - end - return Functors.fmap(get_ps, ps) -end - ## PROBLEM-1 (WITHOUT PARAMETER ESTIMATION) linear_analytic = (u0, p, t) -> u0 + sin(2 * π * t) / (2 * π) linear = (u, p, t) -> cos(2 * π * t) @@ -49,12 +37,10 @@ init1, re1 = destructure(chainflux) θinit, st = Lux.setup(Random.default_rng(), chainlux) fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainflux, - draw_samples = 2500, - n_leapfrog = 30) + draw_samples = 2500) fh_mcmc_chain2, fhsamples2, fhstats2 = ahmc_bayesian_pinn_ode(prob, chainlux, - draw_samples = 2500, - n_leapfrog = 30) + draw_samples = 2500) # can change training strategies by adding this to call (Quadratuer and GridTraining show good results but stochastics sampling techniques perform bad) # strategy = QuadratureTraining(; quadrature_alg = QuadGKJL(), @@ -62,12 +48,10 @@ fh_mcmc_chain2, fhsamples2, fhstats2 = ahmc_bayesian_pinn_ode(prob, chainlux, # abstol = 1e-3, maxiters = 1000, # batch = 0) -alg = NeuralPDE.BNNODE(chainflux, draw_samples = 2500, - n_leapfrog = 30) +alg = NeuralPDE.BNNODE(chainflux, draw_samples = 2500) sol1flux = solve(prob, alg) -alg = NeuralPDE.BNNODE(chainlux, draw_samples = 2500, - n_leapfrog = 30) +alg = NeuralPDE.BNNODE(chainlux, draw_samples = 2500) sol1lux = solve(prob, alg) # testing points @@ -109,9 +93,9 @@ u = sol1.u time = sol1.t # BPINN AND TRAINING DATASET CREATION(dataset must be defined only inside problem timespan!) -ta = range(tspan[1], tspan[2], length = 25) +ta = range(tspan[1], tspan[2], length = 100) u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) .+ (0.2 .* Array(u) .* randn(size(u)))) +x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) time = vec(collect(Float64, ta)) dataset = [x̂, time] physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] @@ -123,10 +107,6 @@ x̂1 = collect(Float64, Array(u1) + 0.2 * randn(size(u1))) time1 = vec(collect(Float64, ta0)) physsol1_1 = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] -using Plots, StatsPlots -# plot(dataset[2], calderivatives(dataset)') -yu = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) -plot(yu, [linear_analytic(u0, p, t) for t in yu]) chainflux1 = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 chainlux1 = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) init1, re1 = destructure(chainflux1) @@ -135,88 +115,37 @@ init1, re1 = destructure(chainflux1) fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainflux1, dataset = dataset, draw_samples = 2500, - physdt = 1 / 50.0f0, + physdt = 1 / 50.0, priorsNNw = (0.0, 3.0), param = [ LogNormal(9, 0.5), - ], - Metric = DiagEuclideanMetric, - n_leapfrog = 30) + ]) fh_mcmc_chain2, fhsamples2, fhstats2 = ahmc_bayesian_pinn_ode(prob, chainlux1, dataset = dataset, draw_samples = 2500, - physdt = 1 / 50.0f0, + physdt = 1 / 50.0, priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30) + param = [LogNormal(9, 0.5)]) alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 10.0), - l2std = [0.005], phystd = [0.01], - param = [Normal(11, 6)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30) -# original paper (pure data 0 1) -sol1flux = solve(prob, alg) -sol1flux.estimated_ode_params -# pure data method 1 1 -sol2flux = solve(prob, alg) -sol2flux.estimated_ode_params -# pure data method 1 0 -sol3flux = solve(prob, alg) -sol3flux.estimated_ode_params -# deri collocation -sol4flux = solve(prob, alg) -sol4flux.estimated_ode_params -# collocation -sol5flux = solve(prob, alg) -sol5flux.estimated_ode_params -# collocation + L2Data loss(at 9,0.5 1,2 gives same) -sol6flux = solve(prob, alg) -sol6flux.estimated_ode_params -# 2500 iters -sol7flux = solve(prob, alg) -sol7flux.estimated_ode_params - -plotly() -plot!(yu, sol1flux.ensemblesol[1]) -plot!(yu, sol2flux.ensemblesol[1]) -plot!(yu, sol3flux.ensemblesol[1]) -plot!(yu, sol4flux.ensemblesol[1]) -plot!(yu, sol5flux.ensemblesol[1]) -plot!(yu, sol6flux.ensemblesol[1]) - -plot!(dataset[2], dataset[1]) - -# plot!(sol4flux.ensemblesol[1]) -# plot!(sol5flux.ensemblesol[1]) - -sol2flux.estimated_ode_params - -sol1flux.estimated_ode_params - -sol3flux.estimated_ode_params - -sol4flux.estimated_ode_params + draw_samples = 2500, physdt = 1 / 50.0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)]) -sol5flux.estimated_ode_params +sol2flux = solve(prob, alg) alg = NeuralPDE.BNNODE(chainlux1, dataset = dataset, draw_samples = 2500, - physdt = 1 / 50.0f0, + physdt = 1 / 50.0, priorsNNw = (0.0, 3.0), param = [ LogNormal(9, 0.5), - ], - Metric = DiagEuclideanMetric, - n_leapfrog = 30) + ]) sol2lux = solve(prob, alg) @@ -238,16 +167,16 @@ meanscurve2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean @test mean(abs.(physsol1 .- meanscurve2)) < 0.15 # ESTIMATED ODE PARAMETERS (NN1 AND NN2) -@test abs(p - mean([fhsamples2[i][23] for i in 2000:2500])) < abs(0.25 * p) -@test abs(p - mean([fhsamples1[i][23] for i in 2000:2500])) < abs(0.25 * p) +@test abs(p - mean([fhsamples2[i][23] for i in 2000:2500])) < abs(0.35 * p) +@test abs(p - mean([fhsamples1[i][23] for i in 2000:2500])) < abs(0.35 * p) #-------------------------- solve() call @test mean(abs.(physsol1_1 .- sol2flux.ensemblesol[1])) < 8e-2 @test mean(abs.(physsol1_1 .- sol2lux.ensemblesol[1])) < 8e-2 # ESTIMATED ODE PARAMETERS (NN1 AND NN2) -@test abs(p - sol1flux.estimated_ode_params[1]) < abs(0.15 * p) -@test abs(p - sol2lux.estimated_ode_params[1]) < abs(0.15 * p) +@test abs(p - sol2flux.estimated_de_params[1]) < abs(0.15 * p) +@test abs(p - sol2lux.estimated_de_params[1]) < abs(0.15 * p) ## PROBLEM-2 linear = (u, p, t) -> u / p + exp(t / p) * cos(t) @@ -277,37 +206,6 @@ chainlux12 = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), Lux.Dense(6 init1, re1 = destructure(chainflux12) θinit, st = Lux.setup(Random.default_rng(), chainlux12) -using Flux -using Random - -function derivatives(chainflux, dataset) - loss(x, y) = Flux.mse(chainflux(x), y) - optimizer = Flux.Optimise.ADAM(0.01) - epochs = 2500 - for epoch in 1:epochs - Flux.train!(loss, Flux.params(chainflux), [(dataset[2]', dataset[1]')], optimizer) - end - getgradient(chainflux, dataset) -end - -function getgradient(chainflux, dataset) - return (chainflux(dataset[end]' .+ sqrt(eps(eltype(Float64)))) .- - chainflux(dataset[end]')) ./ - sqrt(eps(eltype(dataset[end][1]))) -end - -ans = derivatives(chainflux12, dataset) - -init3, re = destructure(chainflux12) -init2 == init1 -init3 == init2 -plot!(dataset[end], ans') -plot!(dataset[end], chainflux12(dataset[end]')') - -ars = getgradient(chainflux12, dataset) - -plot!(dataset[end], ars') - fh_mcmc_chainflux12, fhsamplesflux12, fhstatsflux12 = ahmc_bayesian_pinn_ode(prob, chainflux12, draw_samples = 1500, @@ -315,8 +213,7 @@ fh_mcmc_chainflux12, fhsamplesflux12, fhstatsflux12 = ahmc_bayesian_pinn_ode(pro phystd = [ 0.03], priorsNNw = (0.0, - 10.0), - n_leapfrog = 30) + 10.0)) fh_mcmc_chainflux22, fhsamplesflux22, fhstatsflux22 = ahmc_bayesian_pinn_ode(prob, chainflux12, @@ -331,16 +228,14 @@ fh_mcmc_chainflux22, fhsamplesflux22, fhstatsflux22 = ahmc_bayesian_pinn_ode(pro param = [ Normal(-7, 4), - ], - n_leapfrog = 30) + ]) fh_mcmc_chainlux12, fhsampleslux12, fhstatslux12 = ahmc_bayesian_pinn_ode(prob, chainlux12, draw_samples = 1500, l2std = [0.03], phystd = [0.03], priorsNNw = (0.0, - 10.0), - n_leapfrog = 30) + 10.0)) fh_mcmc_chainlux22, fhsampleslux22, fhstatslux22 = ahmc_bayesian_pinn_ode(prob, chainlux12, dataset = dataset, @@ -352,13 +247,12 @@ fh_mcmc_chainlux22, fhsampleslux22, fhstatslux22 = ahmc_bayesian_pinn_ode(prob, param = [ Normal(-7, 4), - ], - n_leapfrog = 30) + ]) -alg1 = NeuralPDE.BNNODE(chainflux12, +alg = NeuralPDE.BNNODE(chainflux12, dataset = dataset, - draw_samples = 500, - l2std = [0.01], + draw_samples = 1500, + l2std = [0.03], phystd = [ 0.03, ], @@ -367,51 +261,10 @@ alg1 = NeuralPDE.BNNODE(chainflux12, param = [ Normal(-7, 4), - ], - n_leapfrog = 30, progress = true) + ]) -# original paper (pure data 0 1) -sol1flux_pestim = solve(prob, alg1) -sol1flux_pestim.estimated_ode_params -# pure data method 1 1 -sol2flux_pestim = solve(prob, alg1) -sol2flux_pestim.estimated_ode_params -# pure data method 1 0 -sol3flux_pestim = solve(prob, alg1) -sol3flux_pestim.estimated_ode_params -# deri collocation -sol4flux_pestim = solve(prob, alg1) -sol4flux_pestim.estimated_ode_params -# collocation -sol5flux_pestim = solve(prob, alg1) -sol5flux_pestim.estimated_ode_params -# collocation + L2Data loss(at 9,0.5 1,2 gives same) -sol6flux_pestim = solve(prob, alg1) -sol6flux_pestim.estimated_ode_params +sol3flux_pestim = solve(prob, alg) -using Plots, StatsPlots -ars = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) -plot(time, u) -plot!(ars, sol1flux_pestim.ensemblesol[1]) -plot!(ars, sol2flux_pestim.ensemblesol[1]) -plot!(ars, sol3flux_pestim.ensemblesol[1]) -plot!(ars, sol4flux_pestim.ensemblesol[1]) -plot!(ars, sol5flux_pestim.ensemblesol[1]) -plot!(ars, sol6flux_pestim.ensemblesol[1]) - -sol3flux_pestim.estimated_ode_params - -sol4flux_pestim.estimated_ode_params - -sol5flux_pestim.estimated_ode_params - -sol6flux_pestim.estimated_ode_params - -ars = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) - -init, re1 = destructure(chainflux12) -init -init1 alg = NeuralPDE.BNNODE(chainlux12, dataset = dataset, draw_samples = 1500, @@ -422,8 +275,7 @@ alg = NeuralPDE.BNNODE(chainlux12, param = [ Normal(-7, 4), - ], - n_leapfrog = 30) + ]) sol3lux_pestim = solve(prob, alg) @@ -474,3881 +326,11 @@ param1 = mean(i[62] for i in fhsampleslux22[1000:1500]) # (flux chain) @test mean(abs.(physsol2 .- sol3flux_pestim.ensemblesol[1])) < 0.15 # estimated parameters(flux chain) -param1 = sol3flux_pestim.estimated_ode_params[1] +param1 = sol3flux_pestim.estimated_de_params[1] @test abs(param1 - p) < abs(0.45 * p) # (lux chain) @test mean(abs.(physsol2 .- sol3lux_pestim.ensemblesol[1])) < 0.15 # estimated parameters(lux chain) -param1 = sol3lux_pestim.estimated_ode_params[1] -@test abs(param1 - p) < abs(0.45 * p) - -using Plots, StatsPlots -using NoiseRobustDifferentiation, Weave, DataInterpolations - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood -# # 25 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, -# draw_samples = 1500, physdt = 1 / 50.0f0, phystd = [0.01], -# l2std = [0.01], -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux1 = solve(prob, alg) -# sol2flux1.estimated_ode_params[1] #6.41722 Particles{Float64, 1}, 6.02404 Particles{Float64, 1} -# sol2flux2 = solve(prob, alg) -# sol2flux2.estimated_ode_params[1] #6.42782 Particles{Float64, 1}, 6.07509 Particles{Float64, 1} -# sol2flux3 = solve(prob, alg) -# sol2flux3.estimated_ode_params[1] #6.42782 Particles{Float64, 1}, 6.00825 Particles{Float64, 1} - -# # 50 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux11 = solve(prob, alg) -# sol2flux11.estimated_ode_params[1] #5.71268 Particles{Float64, 1}, 6.07242 Particles{Float64, 1} -# sol2flux22 = solve(prob, alg) -# sol2flux22.estimated_ode_params[1] #5.74599 Particles{Float64, 1}, 6.04837 Particles{Float64, 1} -# sol2flux33 = solve(prob, alg) -# sol2flux33.estimated_ode_params[1] #5.74599 Particles{Float64, 1}, 6.02838 Particles{Float64, 1} - -# # 100 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux111 = solve(prob, alg) -# sol2flux111.estimated_ode_params[1] #6.59097 Particles{Float64, 1}, 5.89384 Particles{Float64, 1} -# sol2flux222 = solve(prob, alg) -# sol2flux222.estimated_ode_params[1] #6.62813 Particles{Float64, 1}, 5.88216 Particles{Float64, 1} -# sol2flux333 = solve(prob, alg) -# sol2flux333.estimated_ode_params[1] #6.62813 Particles{Float64, 1}, 5.85327 Particles{Float64, 1} - -# # ---------------------------------------------------------- -# # physdt=1/20, full likelihood cdm -# # 25 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux1_cdm = solve(prob, alg) -# sol2flux1_cdm.estimated_ode_params[1]# 6.50506 Particles{Float64, 1} ,6.38963 Particles{Float64, 1} -# sol2flux2_cdm = solve(prob, alg) -# sol2flux2_cdm.estimated_ode_params[1] #6.50032 Particles{Float64, 1} ,6.39817 Particles{Float64, 1} -# sol2flux3_cdm = solve(prob, alg) -# sol2flux3_cdm.estimated_ode_params[1] #6.50032 Particles{Float64, 1} ,6.36296 Particles{Float64, 1} - -# # 50 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux11_cdm = solve(prob, alg) -# sol2flux11_cdm.estimated_ode_params[1] #6.52951 Particles{Float64, 1},5.15621 Particles{Float64, 1} -# sol2flux22_cdm = solve(prob, alg) -# sol2flux22_cdm.estimated_ode_params[1] #6.54988 Particles{Float64, 1},5.16363 Particles{Float64, 1} -# sol2flux33_cdm = solve(prob, alg) -# sol2flux33_cdm.estimated_ode_params[1] #6.54988 Particles{Float64, 1},5.15591 Particles{Float64, 1} - -# # 100 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux111_cdm = solve(prob, alg) -# sol2flux111_cdm.estimated_ode_params[1] #6.74338 Particles{Float64, 1}, 9.72422 Particles{Float64, 1} -# sol2flux222_cdm = solve(prob, alg) -# sol2flux222_cdm.estimated_ode_params[1] #6.72642 Particles{Float64, 1}, 9.71991 Particles{Float64, 1} -# sol2flux333_cdm = solve(prob, alg) -# sol2flux333_cdm.estimated_ode_params[1] #6.72642 Particles{Float64, 1}, 9.75045 Particles{Float64, 1} - -# -------------------------------------------------------------------------------------- -# NEW SERIES OF TESTS (IN ORDER OF EXECUTION) -# ------------------------------------------------------------------------------------- -# original paper implementaion -# 25 points -ta = range(tspan[1], tspan[2], length = 25) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, u .+ 0.05 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset1 = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] -# scatter!(time, u) -# dataset -# scatter!(dataset1[2], dataset1[1]) -# plot(time, physsol1) - -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_normal = solve(prob, alg) -sol2flux1_normal.estimated_ode_params[1] #7.70593 Particles{Float64, 1}, 6.36096 Particles{Float64, 1} | 6.45865 Particles{Float64, 1} -sol2flux2_normal = solve(prob, alg) -sol2flux2_normal.estimated_ode_params[1] #6.66347 Particles{Float64, 1}, 6.36974 Particles{Float64, 1} | 6.45865 Particles{Float64, 1} -sol2flux3_normal = solve(prob, alg) -sol2flux3_normal.estimated_ode_params[1] #6.84827 Particles{Float64, 1}, 6.29555 Particles{Float64, 1} | 6.39947 Particles{Float64, 1} - -# 50 points -ta = range(tspan[1], tspan[2], length = 50) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.05 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset2 = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_normal = solve(prob, alg) -sol2flux11_normal.estimated_ode_params[1] #7.83577 Particles{Float64, 1},6.24652 Particles{Float64, 1} | 6.34495 Particles{Float64, 1} -sol2flux22_normal = solve(prob, alg) -sol2flux22_normal.estimated_ode_params[1] #6.49477 Particles{Float64, 1},6.2118 Particles{Float64, 1} | 6.32476 Particles{Float64, 1} -sol2flux33_normal = solve(prob, alg) -sol2flux33_normal.estimated_ode_params[1] #6.47421 Particles{Float64, 1},6.33687 Particles{Float64, 1} | 6.2448 Particles{Float64, 1} - -# 100 points -ta = range(tspan[1], tspan[2], length = 100) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.05 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset3 = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_normal = solve(prob, alg) -sol2flux111_normal.estimated_ode_params[1] #5.96604 Particles{Float64, 1},5.99588 Particles{Float64, 1} | 6.19805 Particles{Float64, 1} -sol2flux222_normal = solve(prob, alg) -sol2flux222_normal.estimated_ode_params[1] #6.05432 Particles{Float64, 1},6.0768 Particles{Float64, 1} | 6.22948 Particles{Float64, 1} -sol2flux333_normal = solve(prob, alg) -sol2flux333_normal.estimated_ode_params[1] #6.08856 Particles{Float64, 1},5.94819 Particles{Float64, 1} | 6.2551 Particles{Float64, 1} - -# LOTKA VOLTERRA CASE -function lotka_volterra(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end - -# initial-value problem. -u01 = [1.0, 1.0] -p1 = [1.5, 1.0, 3.0, 1.0] -tspan1 = (0.0, 6.0) -prob1 = ODEProblem(lotka_volterra, u01, tspan1, p1) - -# chainlux = Lux.Chain(Lux.Dense(1, 7, Lux.tanh), Lux.Dense(7, 7, Lux.tanh), Lux.Dense(7, 2)) -chainflux1 = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), Flux.Dense(8, 2)) - -#testing timepoints must match keyword arg `saveat`` timepoints of solve() call -t1 = collect(Float64, prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]) - -# -------------------------------------------------------------------------- -# original paper implementaion lotka volterra -# 31 points -solution1 = solve(prob1, Tsit5(); saveat = 0.1) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] .+ 0.3 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] .+ 0.3 .* u1[2, :] .* randn(length(u1[2, :])) -dataset2_1 = [x1, y1, time1] -plot(dataset2_1[end], dataset2_1[1]) -plot!(dataset2_1[end], dataset2_1[2]) -plot!(time1, u1[1, :]) -plot!(time1, u1[2, :]) - -alg1 = NeuralPDE.BNNODE(chainflux1, - dataset = dataset2_1, - draw_samples = 1000, - physdt = 1 / 20.0, - l2std = [ - 0.2, - 0.2, - ], - phystd = [ - 0.5, - 0.5, - ], - priorsNNw = (0.0, - 10.0), - param = [ - Normal(4, - 3), - Normal(-2, - 4), - Normal(0, - 5), - Normal(2.5, - 2)], - n_leapfrog = 30, progress = true) - -# original paper (pure data 0 1) -sol1flux1_lotka = solve(prob1, alg1) -sol1flux1_lotka.estimated_ode_params -# pure data method 1 1 -sol2flux1_lotka = solve(prob1, alg1) -sol2flux1_lotka.estimated_ode_params -# pure data method 1 0 -sol3flux1_lotka = solve(prob1, alg1) -sol3flux1_lotka.estimated_ode_params -# deri collocation -sol4flux1_lotka = solve(prob1, alg1) -sol4flux1_lotka.estimated_ode_params -# collocation -sol5flux1_lotka = solve(prob1, alg1) -sol5flux1_lotka.estimated_ode_params -# collocation + L2Data loss(at 9,0.5 1,2 gives same) -sol6flux1_lotka = solve(prob1, alg1) -sol6flux1_lotka.estimated_ode_params - -sol7flux1_lotka = solve(prob1, alg1) -sol7flux1_lotka.estimated_ode_params - -using Plots, StatsPlots -plot(dataset2_1[3], u1[1, :]) -plot!(dataset2_1[3], u1[2, :]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol5flux1_normal.ensemblesol[2]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), - sol1flux1_normal.ensemblesol[1], - legend = :outerbottomleft) -sol1flux2_normal = solve(prob1, alg1) -sol1flux2_normal.estimated_ode_params #| -sol1flux3_normal = solve(prob1, alg1) -sol1flux3_normal.estimated_ode_params #| -sol1flux4_normal = solve(prob1, alg1) -sol1flux4_normal.estimated_ode_params - -plotly() -plot!(title = "yuh") -plot!(dataset2_1[3], dataset2_1[1]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux1_normal.ensemblesol[1]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux2_normal.ensemblesol[1]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux3_normal.ensemblesol[2]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux4_normal.ensemblesol[1]) -plot(time1, u1[1, :]) -plot!(time1, u1[2, :]) - -ars = chainflux1(dataset2_1[end]') -plot(ars[1, :]) -plot!(ars[2, :]) - -function calculate_derivatives(dataset) - u = dataset[1] - u1 = dataset[2] - t = dataset[end] - # control points - n = Int(floor(length(t) / 10)) - # spline for datasetvalues(solution) - # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) - interp = CubicSpline(u, t) - interp1 = CubicSpline(u1, t) - # derrivatives interpolation - dx = t[2] - t[1] - time = collect(t[1]:dx:t[end]) - smoothu = [interp(i) for i in time] - smoothu1 = [interp1(i) for i in time] - # derivative of the spline (must match function derivative) - û = tvdiff(smoothu, 20, 0.5, dx = dx, ε = 1) - û1 = tvdiff(smoothu1, 20, 0.5, dx = dx, ε = 1) - # tvdiff(smoothu, 100, 0.035, dx = dx, ε = 1) - # FDM - # û1 = diff(u) / dx - # dataset[1] and smoothu are almost equal(rounding errors) - return û, û1 - # return 1 -end - -ar = calculate_derivatives(dataset2_1) -plot(ar[1]) -plot!(ar[2]) - -# 61 points -solution1 = solve(prob1, Tsit5(); saveat = 0.1) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -dataset2_2 = [x1, y1, time1] - -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_2, - draw_samples = 1000, - l2std = [ - 0.1, - 0.1, - ], - phystd = [ - 0.1, - 0.1, - ], - priorsNNw = (0.0, - 5.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux11_normal = solve(prob1, alg1) -sol1flux11_normal.estimated_ode_params #| -sol1flux22_normal = solve(prob1, alg1) -sol1flux22_normal.estimated_ode_params #| -sol1flux33_normal = solve(prob1, alg1) -sol1flux33_normal.estimated_ode_params #| - -# 121 points -solution1 = solve(prob1, Tsit5(); saveat = 0.05) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -dataset2_3 = [x1, y1, time1] - -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_3, - draw_samples = 1000, - l2std = [ - 0.1, - 0.1, - ], - phystd = [ - 0.1, - 0.1, - ], - priorsNNw = (0.0, - 5.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux111_normal = solve(prob1, alg1) -sol1flux111_normal.estimated_ode_params #| -sol1flux222_normal = solve(prob1, alg1) -sol1flux222_normal.estimated_ode_params #| -sol1flux333_normal = solve(prob1, alg1) -sol1flux333_normal.estimated_ode_params #| - -# -------------------------------------------------------------------- - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# Sampling 100%|███████████████████████████████| Time: 0:02:30 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# Sampling 100%|███████████████████████████████| Time: 0:01:54 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# Sampling 100%|███████████████████████████████| Time: 0:01:59 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# Sampling 100%|███████████████████████████████| Time: 0:02:44 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# Sampling 100%|███████████████████████████████| Time: 0:02:41 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# Sampling 100%|███████████████████████████████| Time: 0:02:41 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# Sampling 100%|███████████████████████████████| Time: 0:03:52 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# Sampling 100%|███████████████████████████████| Time: 0:03:49 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# Sampling 100%|███████████████████████████████| Time: 0:03:50 - -# # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> -# physics Logpdf is : -6.659143464386241e7 -# prior Logpdf is : -150.30074579848434 -# L2lossData Logpdf is : -6.03075717462954e6 -# Sampling 100%|███████████████████████████████| Time: 0:04:54 - -# physics Logpdf is : -8.70012053004202e8 -# prior Logpdf is : -150.3750892952511 -# L2lossData Logpdf is : -6.967914805207133e6 -# Sampling 100%|███████████████████████████████| Time: 0:05:09 - -# physics Logpdf is : -5.417241281343099e7 -# prior Logpdf is : -150.52079555737976 -# L2lossData Logpdf is : -4.195953436792884e6 -# Sampling 100%|███████████████████████████████| Time: 0:05:01 - -# physics Logpdf is : -4.579552981943833e8 -# prior Logpdf is : -150.30491731974283 -# L2lossData Logpdf is : -8.595475827260146e6 -# Sampling 100%|███████████████████████████████| Time: 0:06:08 - -# physics Logpdf is : -1.989281834955769e7 -# prior Logpdf is : -150.16009042727543 -# L2lossData Logpdf is : -1.121270659669029e7 -# Sampling 100%|███████████████████████████████| Time: 0:05:38 - -# physics Logpdf is : -8.683829147264534e8 -# prior Logpdf is : -150.37824872259102 -# L2lossData Logpdf is : -1.0887662888035845e7 -# Sampling 100%|███████████████████████████████| Time: 0:05:50 - -# physics Logpdf is : -3.1944760610332566e8 -# prior Logpdf is : -150.33610348737565 -# L2lossData Logpdf is : -1.215458786744478e7 -# Sampling 100%|███████████████████████████████| Time: 0:10:50 - -# physics Logpdf is : -3.2884572300341567e6 -# prior Logpdf is : -150.21002268156343 -# L2lossData Logpdf is : -1.102536731511176e7 -# Sampling 100%|███████████████████████████████| Time: 0:09:53 - -# physics Logpdf is : -5.31293521002414e8 -# prior Logpdf is : -150.20948536040126 -# L2lossData Logpdf is : -1.818717239584132e7 -# Sampling 100%|███████████████████████████████| Time: 0:08:53 - -# ---------------------------------------------------------- -# Full likelihood no l2 only new L22(NN gradients) -# 25 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_new = solve(prob, alg) -sol2flux1_new.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 6.21662 Particles{Float64, 1} -sol2flux2_new = solve(prob, alg) -sol2flux2_new.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 7.14238 Particles{Float64, 1} -sol2flux3_new = solve(prob, alg) -sol2flux3_new.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 6.79159 Particles{Float64, 1} - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_new = solve(prob, alg) -sol2flux11_new.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 5.33467 Particles{Float64, 1} -sol2flux22_new = solve(prob, alg) -sol2flux22_new.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.52419 Particles{Float64, 1} -sol2flux33_new = solve(prob, alg) -sol2flux33_new.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 5.36921 Particles{Float64, 1} - -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_new = solve(prob, alg) -sol2flux111_new.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.45333 Particles{Float64, 1} -sol2flux222_new = solve(prob, alg) -sol2flux222_new.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 4.64417 Particles{Float64, 1} -sol2flux333_new = solve(prob, alg) -sol2flux333_new.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | 5.88037 Particles{Float64, 1} -# --------------------------------------------------------------------------- - -# ---------------------------------------------------------- -# Full likelihood l2 + new L22(NN gradients) -# 25 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_new_all = solve(prob, alg) -sol2flux1_new_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 6.4358 Particles{Float64, 1} -sol2flux2_new_all = solve(prob, alg) -sol2flux2_new_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 6.52449 Particles{Float64, 1} -sol2flux3_new_all = solve(prob, alg) -sol2flux3_new_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 6.34188 Particles{Float64, 1} - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_new_all = solve(prob, alg) -sol2flux11_new_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 6.37889 Particles{Float64, 1} -sol2flux22_new_all = solve(prob, alg) -sol2flux22_new_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.34747 Particles{Float64, 1} -sol2flux33_new_all = solve(prob, alg) -sol2flux33_new_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 6.39699 Particles{Float64, 1} - -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_new_all = solve(prob, alg) -sol2flux111_new_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.24327 Particles{Float64, 1} -sol2flux222_new_all = solve(prob, alg) -sol2flux222_new_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 6.23928 Particles{Float64, 1} -sol2flux333_new_all = solve(prob, alg) -sol2flux333_new_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | 6.2145 Particles{Float64, 1} - -# --------------------------------------------------------------------------- -# Full likelihood l2 + new L22(dataset gradients) lotka volterra -# 36 points -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_1, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux1_new_all = solve(prob1, alg1) -sol1flux1_new_all.estimated_ode_params[1] #| -sol1flux2_new_all = solve(prob1, alg1) -sol1flux2_new_all.estimated_ode_params[1] #| -sol1flux3_new_all = solve(prob1, alg1) -sol1flux3_new_all.estimated_ode_params[1] #| - -# 61 points -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_2, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux11_new_all = solve(prob1, alg1) -sol1flux11_new_all.estimated_ode_params[1] #| -sol1flux22_new_all = solve(prob1, alg1) -sol1flux22_new_all.estimated_ode_params[1] #| -sol1flux33_new_all = solve(prob1, alg1) -sol1flux33_new_all.estimated_ode_params[1] #| - -# 121 points -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_3, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux111_new_all = solve(prob1, alg1) -sol1flux111_new_all.estimated_ode_params[1] #| -sol1flux222_new_all = solve(prob1, alg1) -sol1flux222_new_all.estimated_ode_params[1] #| -sol1flux333_new_all = solve(prob1, alg1) -sol1flux333_new_all.estimated_ode_params[1] #| -# -------------------------------------------------------------------- - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# L2loss2 Logpdf is : -757.9047847584478 -# Sampling 100%|███████████████████████████████| Time: 0:02:32 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# L2loss2 Logpdf is : -757.9047847584478 -# Sampling 100%|███████████████████████████████| Time: 0:02:19 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# L2loss2 Logpdf is : -757.9047847584478 -# Sampling 100%|███████████████████████████████| Time: 0:02:31 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# L2loss2 Logpdf is : -1517.3653615845183 -# Sampling 100%|███████████████████████████████| Time: 0:03:45 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# L2loss2 Logpdf is : -1517.3653615845183 -# Sampling 100%|███████████████████████████████| Time: 0:03:20 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# L2loss2 Logpdf is : -1517.3653615845183 -# Sampling 100%|███████████████████████████████| Time: 0:03:20 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# L2loss2 Logpdf is : -3037.8868319811254 -# Sampling 100%|███████████████████████████████| Time: 0:04:57 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# L2loss2 Logpdf is : -3037.8868319811254 -# Sampling 100%|███████████████████████████████| Time: 0:05:26 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# L2loss2 Logpdf is : -3037.8868319811254 -# Sampling 100%|███████████████████████████████| Time: 0:05:01 - -# ---------------------------------------------------------- -# Full likelihood l2 + new L22(dataset gradients) -# 25 points -# 1*,2*, -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_newdata_all = solve(prob, alg) -sol2flux1_newdata_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 5.73072 Particles{Float64, 1} -sol2flux2_newdata_all = solve(prob, alg) -sol2flux2_newdata_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 5.71597 Particles{Float64, 1} -sol2flux3_newdata_all = solve(prob, alg) -sol2flux3_newdata_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 5.7313 Particles{Float64, 1} - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_newdata_all = solve(prob, alg) -sol2flux11_newdata_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 6.07153 Particles{Float64, 1} -sol2flux22_newdata_all = solve(prob, alg) -sol2flux22_newdata_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.06623 Particles{Float64, 1} -sol2flux33_newdata_all = solve(prob, alg) -sol2flux33_newdata_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 6.12748 Particles{Float64, 1} - -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_newdata_all = solve(prob, alg) -sol2flux111_newdata_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.26222 Particles{Float64, 1} -sol2flux222_newdata_all = solve(prob, alg) -sol2flux222_newdata_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 5.86494 Particles{Float64, 1} -sol2flux333_newdata_all = solve(prob, alg) -sol2flux333_newdata_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | - -# --------------------------------------------------------------------------- - -# LOTKA VOLTERRA CASE -using Plots, StatsPlots -function lotka_volterra(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end - -# initial-value problem. -u01 = [1.0, 1.0] -p1 = [1.5, 1.0, 3.0, 1.0] -tspan1 = (0.0, 6.0) -prob1 = ODEProblem(lotka_volterra, u01, tspan1, p1) - -chainlux = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), Lux.Dense(6, 2)) - -#testing timepoints must match keyword arg `saveat`` timepoints of solve() call -t1 = collect(Float64, prob.tspan[1]:(1 / 50.0):prob.tspan[2]) - -# -------------------------------------------------------------------------- -# original paper implementaion -# 25 points -solution1 = solve(prob1, Tsit5(); saveat = 0.2) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -dataset2_1 = [x1, y1, time1] - -plot(time1, u1[1, :]) -plot!(time1, u1[2, :]) -scatter!(dataset2_1[3], dataset2_1[1]) -scatter!(dataset2_1[3], dataset2_1[2]) - -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_1, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux1_normal = solve(prob1, alg1) -sol1flux1_normal.estimated_ode_params[1] #| -sol1flux2_normal = solve(prob1, alg1) -sol1flux2_normal.estimated_ode_params[1] #| -sol1flux3_normal = solve(prob1, alg1) -sol1flux3_normal.estimated_ode_params[1] #| - -# 50 points -solution1 = solve(prob1, Tsit5(); saveat = 0.05) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -dataset2_2 = [x1, y1, time1] - -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_2, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux11_normal = solve(prob1, alg1) -sol1flux11_normal.estimated_ode_params[1] #| -sol1flux22_normal = solve(prob1, alg1) -sol1flux22_normal.estimated_ode_params[1] #| -sol1flux33_normal = solve(prob1, alg1) -sol1flux33_normal.estimated_ode_params[1] #| - -# 100 points -solution = solve(prob1, Tsit5(); saveat = 0.05) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -dataset2_3 = [x1, y1, time1] - -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_3, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux111_normal = solve(prob1, alg1) -sol1flux111_normal.estimated_ode_params[1] #| -sol1flux222_normal = solve(prob1, alg1) -sol1flux222_normal.estimated_ode_params[1] #| -sol1flux333_normal = solve(prob1, alg1) -sol1flux333_normal.estimated_ode_params[1] #| - -# -------------------------------------------------------------------- - -# ---------------------------------------------------------- -# Full likelihood no l2 only new L22(NN gradients) -# 25 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_new = solve(prob, alg) -sol2flux1_new.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | -sol2flux2_new = solve(prob, alg) -sol2flux2_new.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | -sol2flux3_new = solve(prob, alg) -sol2flux3_new.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_new = solve(prob, alg) -sol2flux11_new.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | -sol2flux22_new = solve(prob, alg) -sol2flux22_new.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | -sol2flux33_new = solve(prob, alg) -sol2flux33_new.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | - -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_new = solve(prob, alg) -sol2flux111_new.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | -sol2flux222_new = solve(prob, alg) -sol2flux222_new.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | -sol2flux333_new = solve(prob, alg) -sol2flux333_new.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | -# --------------------------------------------------------------------------- - -# ---------------------------------------------------------- -# Full likelihood l2 + new L22(NN gradients) -# 25 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_new_all = solve(prob, alg) -sol2flux1_new_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | -sol2flux2_new_all = solve(prob, alg) -sol2flux2_new_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | -sol2flux3_new_all = solve(prob, alg) -sol2flux3_new_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_new_all = solve(prob, alg) -sol2flux11_new_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | -sol2flux22_new_all = solve(prob, alg) -sol2flux22_new_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | -sol2flux33_new_all = solve(prob, alg) -sol2flux33_new_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_new_all = solve(prob, alg) -sol2flux111_new_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | -sol2flux222_new_all = solve(prob, alg) -sol2flux222_new_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | -sol2flux333_new_all = solve(prob, alg) -sol2flux333_new_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | - -# --------------------------------------------------------------------------- - -# ---------------------------------------------------------- -# Full likelihood l2 + new L22(dataset gradients) -# 25 points -# *1,*2 vs *2.5 -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol1flux1_newdata_all = solve(prob, alg) -sol1flux1_newdata_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | -sol1flux2_newdata_all = solve(prob, alg) -sol1flux2_newdata_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | -sol1flux3_newdata_all = solve(prob, alg) -sol1flux3_newdata_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol1flux11_newdata_all = solve(prob, alg) -sol1flux11_newdata_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | -sol1flux22_newdata_all = solve(prob, alg) -sol1flux22_newdata_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | -sol1flux33_newdata_all = solve(prob, alg) -sol1flux33_newdata_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | - -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol1flux111_newdata_all = solve(prob, alg) -sol1flux111_newdata_all.estimated_ode_params[1] #| -sol1flux222_newdata_all = solve(prob, alg) -sol1flux222_newdata_all.estimated_ode_params[1] #| -sol1flux333_newdata_all = solve(prob, alg) -sol1flux333_newdata_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | - -# ------------------------------------------------------------------------------------------------------------------------------ - -# sol2flux111.estimated_ode_params[1] -# # mine *5 -# 7.03386Particles{Float64, 1} -# # normal -# 6.38951Particles{Float64, 1} -# 6.67657Particles{Float64, 1} -# # mine *10 -# 7.53672Particles{Float64, 1} -# # mine *2 -# 6.29005Particles{Float64, 1} -# 6.29844Particles{Float64, 1} - -# # new mine *2 -# 6.39008Particles{Float64, 1} -# 6.22071Particles{Float64, 1} -# 6.15611Particles{Float64, 1} - -# # new mine *2 tvdiff(smoothu, 20, 0.035, dx = dx, ε = 1e-2) -# 6.25549Particles{Float64, 1} -# ---------------------------------------------------------- - -# --------------------------------------------------- - -function calculate_derivatives1(dataset) - x̂, time = dataset - num_points = length(x̂) - # Initialize an array to store the derivative values. - derivatives = similar(x̂) - - for i in 2:(num_points - 1) - # Calculate the first-order derivative using central differences. - Δt_forward = time[i + 1] - time[i] - Δt_backward = time[i] - time[i - 1] - - derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) - - derivatives[i] = derivative - end - - # Derivatives at the endpoints can be calculated using forward or backward differences. - derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) - derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) - return derivatives -end - -function calculate_derivatives2(dataset) - u = dataset[1] - t = dataset[2] - # control points - n = Int(floor(length(t) / 10)) - # spline for datasetvalues(solution) - # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) - interp = CubicSpline(u, t) - # derrivatives interpolation - dx = t[2] - t[1] - time = collect(t[1]:dx:t[end]) - smoothu = [interp(i) for i in time] - # derivative of the spline (must match function derivative) - û = tvdiff(smoothu, 20, 0.03, dx = dx, ε = 1) - # tvdiff(smoothu, 100, 0.1, dx = dx) - # - # - # FDM - û1 = diff(u) / dx - # dataset[1] and smoothu are almost equal(rounding errors) - return û, time, smoothu, û1 -end - -# need to do this for all datasets -c = [linear(prob.u0, p, t) for t in dataset3[2]] #ideal case -b = calculate_derivatives1(dataset2) #central diffs -# a = calculate_derivatives2(dataset) #tvdiff(smoothu, 100, 0.1, dx = dx) -d = calculate_derivatives2(dataset1) #tvdiff(smoothu, 20, 0.035, dx = dx, ε = 1e-2) -d = calculate_derivatives2(dataset2) -d = calculate_derivatives2(dataset3) -mean(abs2.(c .- b)) -mean(abs2.(c .- d[1])) -loss(model, x, y) = mean(abs2.(model(x) .- y)); -scatter!(prob.u0 .+ (prob.tspan[2] .- dataset3[2]) .* chainflux1(dataset3[2]')') -loss(chainflux1, dataset3[2]', dataset3[1]') -# mean(abs2.(c[1:24] .- a[4])) -plot(c, label = "ideal deriv") -plot!(b, label = "Centraldiff deriv") -# plot!(a[1], label = "tvdiff(0.1,def) derivatives") -plot!(d[1], label = "tvdiff(0.035,20) derivatives") -plotly() - -# GridTraining , NoiseRobustDiff dataset[2][2]-dataset[2][1] l2std -# 25 points -ta = range(tspan[1], tspan[2], length = 25) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -time1 = collect(tspan[1]:(1 / 50.0):tspan[2]) -physsol = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] -plot(physsol, label = "solution") - -# plots from 32(deriv) -# for d -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 2000, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux1 = solve(prob, alg) -n2_sol2flux1.estimated_ode_params[1] -# with extra likelihood -# 10.2011Particles{Float64, 1} - -# without extra likelihood -# 6.25791Particles{Float64, 1} -# 6.29539Particles{Float64, 1} - -plot!(n2_sol2flux1.ensemblesol[1], label = "tvdiff(0.035,1) derivpar") -plot(dataset[1]) -plot!(physsol1) -# for a -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 2000, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux2 = solve(prob, alg) -n2_sol2flux2.estimated_ode_params[1] -# with extra likelihood -# 8.73602Particles{Float64, 1} -# without extra likelihood - -plot!(n2_sol2flux2.ensemblesol[1], - label = "tvdiff(0.1,def) derivatives", - legend = :outerbottomleft) - -# for b -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 2000, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux3 = solve(prob, alg) -n2_sol2flux3.estimated_ode_params[1] -plot!(n2_sol2flux3.ensemblesol[1], label = "Centraldiff deriv") - -# for c -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 2000, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux4 = solve(prob, alg) -n2_sol2flux4.estimated_ode_params[1] -plot!(n2_sol2flux4.ensemblesol[1], label = "ideal deriv") - -# 50 points - -ta = range(tspan[1], tspan[2], length = 50) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux11 = solve(prob, alg) -n2_sol2flux11.estimated_ode_params[1] - -# 5.90049Particles{Float64, 1} -# 100 points -ta = range(tspan[1], tspan[2], length = 100) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux111 = solve(prob, alg) -n2_sol2flux111.estimated_ode_params[1] -plot!(n2_sol2flux111.ensemblesol[1]) -8.88555Particles{Float64, 1} - -# 7.15353Particles{Float64, 1} -# 6.21059 Particles{Float64, 1} -# 6.31836Particles{Float64, 1} -0.1 * p -# ---------------------------------------------------------- - -# Gives the linear interpolation value at t=3.5 - -# # Problem 1 with param esimation -# # dataset 0-1 2 percent noise -# p = 6.283185307179586 -# # partial_logdensity -# 6.3549Particles{Float64, 1} -# # full log_density -# 6.34667Particles{Float64, 1} - -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2lux.estimated_ode_params[1] - -# # dataset 0-1 20 percent noise -# # partial log_density -# 6.30244Particles{Float64, 1} -# # full log_density -# 6.24637Particles{Float64, 1} - -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # dataset 0-2 20percent noise -# # partial log_density -# 6.24948Particles{Float64, 1} -# # full log_density -# 6.26095Particles{Float64, 1} - -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# linear_analytic = (u0, p, t) -> u0 + sin(p * t) / (p) -# linear = (u, p, t) -> cos(p * t) -# tspan = (0.0, 2.0) - -# # dataset 0-1 2 percent noise -# p = 6.283185307179586 -# # partial_logdensity -# 6.3549Particles{Float64, 1} -# # full log_density -# 6.34667Particles{Float64, 1} - -# # dataset 0-1 20 percent noise -# # partial log_density -# 6.30244Particles{Float64, 1} -# # full log_density -# 6.24637Particles{Float64, 1} - -# # dataset 0-2 20percent noise -# # partial log_density -# 6.24948Particles{Float64, 1} -# # full log_density -# 6.26095Particles{Float64, 1} - -# # dataset 0-2 20percent noise 50 points(above all are 100 points) -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # partial log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# # i kinda win on 25 points again -# # dataset 0-2 20percent noise 25 points -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # partial log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # i win with 25 points -# # dataset 0-1 20percent noise 25 points -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# # new -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # partial log_density -# sol2flux.estimated_ode_params[1] -# # New -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # (9,2.5)(above are (9,0.5)) -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# # just prev was repeat(just change) -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # partial log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # i lose on 0-1,50 points -# # dataset 0-1 20percent noise 50 points -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# # partial log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # (9,2.5) (above are (9,0.5)) -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # partial log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # ---------------------------------------------------------- -# # Problem 1 with param estimation -# # physdt=1/20, Full likelihood new 0.5*l2std -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n05_sol2flux1 = solve(prob, alg) -# n05_sol2flux1.estimated_ode_params[1] #6.90953 Particles{Float64, 1} -# n05_sol2flux2 = solve(prob, alg) -# n05_sol2flux2.estimated_ode_params[1] #6.82374 Particles{Float64, 1} -# n05_sol2flux3 = solve(prob, alg) -# n05_sol2flux3.estimated_ode_params[1] #6.84465 Particles{Float64, 1} - -# using Plots, StatsPlots -# plot(n05_sol2flux3.ensemblesol[1]) -# plot!(physsol1) -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n05_sol2flux11 = solve(prob, alg) -# n05_sol2flux11.estimated_ode_params[1] #7.0262 Particles{Float64, 1} -# n05_sol2flux22 = solve(prob, alg) -# n05_sol2flux22.estimated_ode_params[1] #5.56438 Particles{Float64, 1} -# n05_sol2flux33 = solve(prob, alg) -# n05_sol2flux33.estimated_ode_params[1] #7.27189 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n05_sol2flux111 = solve(prob, alg) -# n05_sol2flux111.estimated_ode_params[1] #6.90549 Particles{Float64, 1} -# n05_sol2flux222 = solve(prob, alg) -# n05_sol2flux222.estimated_ode_params[1] #5.42436 Particles{Float64, 1} -# n05_sol2flux333 = solve(prob, alg) -# n05_sol2flux333.estimated_ode_params[1] #6.05832 Particles{Float64, 1} - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood new 2*l2std -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2_sol2flux1 = solve(prob, alg) -# n2_sol2flux1.estimated_ode_params[1]#6.9087 Particles{Float64, 1} -# n2_sol2flux2 = solve(prob, alg) -# n2_sol2flux2.estimated_ode_params[1]#6.86507 Particles{Float64, 1} -# n2_sol2flux3 = solve(prob, alg) -# n2_sol2flux3.estimated_ode_params[1]#6.59206 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2_sol2flux11 = solve(prob, alg) -# n2_sol2flux11.estimated_ode_params[1]#7.3715 Particles{Float64, 1} -# n2_sol2flux22 = solve(prob, alg) -# n2_sol2flux22.estimated_ode_params[1]#9.84477 Particles{Float64, 1} -# n2_sol2flux33 = solve(prob, alg) -# n2_sol2flux33.estimated_ode_params[1]#6.87107 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2_sol2flux111 = solve(prob, alg) -# n2_sol2flux111.estimated_ode_params[1]#6.60739 Particles{Float64, 1} -# n2_sol2flux222 = solve(prob, alg) -# n2_sol2flux222.estimated_ode_params[1]#7.05923 Particles{Float64, 1} -# n2_sol2flux333 = solve(prob, alg) -# n2_sol2flux333.estimated_ode_params[1]#6.5017 Particles{Float64, 1} - -# # ---------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood new all 2*l2std -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2all5sol2flux1 = solve(prob, alg) -# n2all5sol2flux1.estimated_ode_params[1]#11.3659 Particles{Float64, 1} -# n2all5sol2flux2 = solve(prob, alg) -# n2all5sol2flux2.estimated_ode_params[1]#6.65634 Particles{Float64, 1} -# n2all5sol2flux3 = solve(prob, alg) -# n2all5sol2flux3.estimated_ode_params[1]#6.61905 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2all5sol2flux11 = solve(prob, alg) -# n2all5sol2flux11.estimated_ode_params[1]#6.27555 Particles{Float64, 1} -# n2all5sol2flux22 = solve(prob, alg) -# n2all5sol2flux22.estimated_ode_params[1]#6.24352 Particles{Float64, 1} -# n2all5sol2flux33 = solve(prob, alg) -# n2all5sol2flux33.estimated_ode_params[1]#6.33723 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2all5sol2flux111 = solve(prob, alg) -# n2all5sol2flux111.estimated_ode_params[1] #5.95535 Particles{Float64, 1} -# n2all5sol2flux222 = solve(prob, alg) -# n2all5sol2flux222.estimated_ode_params[1] #5.98301 Particles{Float64, 1} -# n2all5sol2flux333 = solve(prob, alg) -# n2all5sol2flux333.estimated_ode_params[1] #5.9081 Particles{Float64, 1} - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood new all (l2+l22) -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nall5sol2flux1 = solve(prob, alg) -# nall5sol2flux1.estimated_ode_params[1]#6.54705 Particles{Float64, 1} -# nall5sol2flux2 = solve(prob, alg) -# nall5sol2flux2.estimated_ode_params[1]#6.6967 Particles{Float64, 1} -# nall5sol2flux3 = solve(prob, alg) -# nall5sol2flux3.estimated_ode_params[1]#6.47173 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nall5sol2flux11 = solve(prob, alg) -# nall5sol2flux11.estimated_ode_params[1]#6.2113 Particles{Float64, 1} -# nall5sol2flux22 = solve(prob, alg) -# nall5sol2flux22.estimated_ode_params[1]#6.10675 Particles{Float64, 1} -# nall5sol2flux33 = solve(prob, alg) -# nall5sol2flux33.estimated_ode_params[1]#6.11541 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nall5sol2flux111 = solve(prob, alg) -# nall5sol2flux111.estimated_ode_params[1]#6.35224 Particles{Float64, 1} -# nall5sol2flux222 = solve(prob, alg) -# nall5sol2flux222.estimated_ode_params[1]#6.40542 Particles{Float64, 1} -# nall5sol2flux333 = solve(prob, alg) -# nall5sol2flux333.estimated_ode_params[1]#6.44206 Particles{Float64, 1} - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood new 5* (new only l22 mod) -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n5sol2flux1 = solve(prob, alg) -# n5sol2flux1.estimated_ode_params[1]#7.05077 Particles{Float64, 1} -# n5sol2flux2 = solve(prob, alg) -# n5sol2flux2.estimated_ode_params[1]#7.07303 Particles{Float64, 1} -# n5sol2flux3 = solve(prob, alg) -# n5sol2flux3.estimated_ode_params[1]#5.10622 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n5sol2flux11 = solve(prob, alg) -# n5sol2flux11.estimated_ode_params[1]#7.39852 Particles{Float64, 1} -# n5sol2flux22 = solve(prob, alg) -# n5sol2flux22.estimated_ode_params[1]#7.30319 Particles{Float64, 1} -# n5sol2flux33 = solve(prob, alg) -# n5sol2flux33.estimated_ode_params[1]#6.73722 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n5sol2flux111 = solve(prob, alg) -# n5sol2flux111.estimated_ode_params[1]#7.15996 Particles{Float64, 1} -# n5sol2flux222 = solve(prob, alg) -# n5sol2flux222.estimated_ode_params[1]#7.02949 Particles{Float64, 1} -# n5sol2flux333 = solve(prob, alg) -# n5sol2flux333.estimated_ode_params[1]#6.9393 Particles{Float64, 1} - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood new -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nsol2flux1 = solve(prob, alg) -# nsol2flux1.estimated_ode_params[1] #5.82707 Particles{Float64, 1} -# nsol2flux2 = solve(prob, alg) -# nsol2flux2.estimated_ode_params[1] #4.81534 Particles{Float64, 1} -# nsol2flux3 = solve(prob, alg) -# nsol2flux3.estimated_ode_params[1] #5.52965 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nsol2flux11 = solve(prob, alg) -# nsol2flux11.estimated_ode_params[1] #7.04027 Particles{Float64, 1} -# nsol2flux22 = solve(prob, alg) -# nsol2flux22.estimated_ode_params[1] #7.17588 Particles{Float64, 1} -# nsol2flux33 = solve(prob, alg) -# nsol2flux33.estimated_ode_params[1] #6.94495 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nsol2flux111 = solve(prob, alg) -# nsol2flux111.estimated_ode_params[1] #6.06608 Particles{Float64, 1} -# nsol2flux222 = solve(prob, alg) -# nsol2flux222.estimated_ode_params[1] #6.84726 Particles{Float64, 1} -# nsol2flux333 = solve(prob, alg) -# nsol2flux333.estimated_ode_params[1] #6.83463 Particles{Float64, 1} - -# # ---------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux1 = solve(prob, alg) -# sol2flux1.estimated_ode_params[1] #6.71397 Particles{Float64, 1} 6.37604 Particles{Float64, 1} -# sol2flux2 = solve(prob, alg) -# sol2flux2.estimated_ode_params[1] #6.73509 Particles{Float64, 1} 6.21692 Particles{Float64, 1} -# sol2flux3 = solve(prob, alg) -# sol2flux3.estimated_ode_params[1] #6.65453 Particles{Float64, 1} 6.23153 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux11 = solve(prob, alg) -# sol2flux11.estimated_ode_params[1] #6.23443 Particles{Float64, 1} 6.30635 Particles{Float64, 1} -# sol2flux22 = solve(prob, alg) -# sol2flux22.estimated_ode_params[1] #6.18879 Particles{Float64, 1} 6.30099 Particles{Float64, 1} -# sol2flux33 = solve(prob, alg) -# sol2flux33.estimated_ode_params[1] #6.22773 Particles{Float64, 1} 6.30671 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux111 = solve(prob, alg) -# sol2flux111.estimated_ode_params[1] #6.15832 Particles{Float64, 1} 6.35453 Particles{Float64, 1} -# sol2flux222 = solve(prob, alg) -# sol2flux222.estimated_ode_params[1] #6.16968 Particles{Float64, 1}6.31125 Particles{Float64, 1} -# sol2flux333 = solve(prob, alg) -# sol2flux333.estimated_ode_params[1] #6.12466 Particles{Float64, 1} 6.26514 Particles{Float64, 1} - -# # ---------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, partial likelihood -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux1_p = solve(prob, alg) -# sol2flux1_p.estimated_ode_params[1] #5.74065 Particles{Float64, 1} #6.83683 Particles{Float64, 1} -# sol2flux2_p = solve(prob, alg) -# sol2flux2_p.estimated_ode_params[1] #9.82504 Particles{Float64, 1} #6.14568 Particles{Float64, 1} -# sol2flux3_p = solve(prob, alg) -# sol2flux3_p.estimated_ode_params[1] #5.75075 Particles{Float64, 1} #6.08579 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux11_p = solve(prob, alg) -# sol2flux11_p.estimated_ode_params[1] #6.19414 Particles{Float64, 1} #6.04621 Particles{Float64, 1} -# sol2flux22_p = solve(prob, alg) -# sol2flux22_p.estimated_ode_params[1] #6.15227 Particles{Float64, 1} #6.29086 Particles{Float64, 1} -# sol2flux33_p = solve(prob, alg) -# sol2flux33_p.estimated_ode_params[1] #6.19048 Particles{Float64, 1} #6.12516 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux111_p = solve(prob, alg) -# sol2flux111_p.estimated_ode_params[1] #6.51608 Particles{Float64, 1}# 6.42945Particles{Float64, 1} -# sol2flux222_p = solve(prob, alg) -# sol2flux222_p.estimated_ode_params[1] #6.4875 Particles{Float64, 1} # 6.44524Particles{Float64, 1} -# sol2flux333_p = solve(prob, alg) -# sol2flux333_p.estimated_ode_params[1] #6.51679 Particles{Float64, 1}# 6.43152Particles{Float64, 1} - -# # --------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood, dataset(1.0-2.0) -# # 25 points -# ta = range(1.0, tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux1 = solve(prob, alg) -# sol1flux1.estimated_ode_params[1] #6.35164 Particles{Float64, 1} -# sol1flux2 = solve(prob, alg) -# sol1flux2.estimated_ode_params[1] #6.30919 Particles{Float64, 1} -# sol1flux3 = solve(prob, alg) -# sol1flux3.estimated_ode_params[1] #6.33554 Particles{Float64, 1} - -# # 50 points -# ta = range(1.0, tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux11 = solve(prob, alg) -# sol1flux11.estimated_ode_params[1] #6.39769 Particles{Float64, 1} -# sol1flux22 = solve(prob, alg) -# sol1flux22.estimated_ode_params[1] #6.43924 Particles{Float64, 1} -# sol1flux33 = solve(prob, alg) -# sol1flux33.estimated_ode_params[1] #6.4697 Particles{Float64, 1} - -# # 100 points -# ta = range(1.0, tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux111 = solve(prob, alg) -# sol1flux111.estimated_ode_params[1] #6.27812 Particles{Float64, 1} -# sol1flux222 = solve(prob, alg) -# sol1flux222.estimated_ode_params[1] #6.19278 Particles{Float64, 1} -# sol1flux333 = solve(prob, alg) -# sol1flux333.estimated_ode_params[1] # 9.68244Particles{Float64, 1} (first try) # 6.23969 Particles{Float64, 1}(second try) - -# # ---------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, partial likelihood, dataset(1.0-2.0) -# # 25 points -# ta = range(1.0, tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux1_p = solve(prob, alg) -# sol1flux1_p.estimated_ode_params[1]#6.36269 Particles{Float64, 1} - -# sol1flux2_p = solve(prob, alg) -# sol1flux2_p.estimated_ode_params[1]#6.34685 Particles{Float64, 1} - -# sol1flux3_p = solve(prob, alg) -# sol1flux3_p.estimated_ode_params[1]#6.31421 Particles{Float64, 1} - -# # 50 points -# ta = range(1.0, tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux11_p = solve(prob, alg) -# sol1flux11_p.estimated_ode_params[1] #6.15725 Particles{Float64, 1} - -# sol1flux22_p = solve(prob, alg) -# sol1flux22_p.estimated_ode_params[1] #6.18145 Particles{Float64, 1} - -# sol1flux33_p = solve(prob, alg) -# sol1flux33_p.estimated_ode_params[1] #6.21905 Particles{Float64, 1} - -# # 100 points -# ta = range(1.0, tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux111_p = solve(prob, alg) -# sol1flux111_p.estimated_ode_params[1]#6.13481 Particles{Float64, 1} - -# sol1flux222_p = solve(prob, alg) -# sol1flux222_p.estimated_ode_params[1]#9.68555 Particles{Float64, 1} - -# sol1flux333_p = solve(prob, alg) -# sol1flux333_p.estimated_ode_params[1]#6.1477 Particles{Float64, 1} - -# # ----------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, partial likelihood, dataset(1-2), again but different density -# # 12 points -# ta = range(1.0, tspan[2], length = 12) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol3flux1_p = solve(prob, alg) -# sol3flux1_p.estimated_ode_params[1]#6.50048 Particles{Float64, 1} -# sol3flux2_p = solve(prob, alg) -# sol3flux2_p.estimated_ode_params[1]#6.57597 Particles{Float64, 1} -# sol3flux3_p = solve(prob, alg) -# sol3flux3_p.estimated_ode_params[1]#6.24487 Particles{Float64, 1} - -# # 25 points -# ta = range(1.0, tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol3flux11_p = solve(prob, alg) -# sol3flux11_p.estimated_ode_params[1]#6.53093 Particles{Float64, 1} - -# sol3flux22_p = solve(prob, alg) -# sol3flux22_p.estimated_ode_params[1]#6.32744 Particles{Float64, 1} - -# sol3flux33_p = solve(prob, alg) -# sol3flux33_p.estimated_ode_params[1]#6.49175 Particles{Float64, 1} - -# # 50 points -# ta = range(1.0, tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol3flux111_p = solve(prob, alg) -# sol3flux111_p.estimated_ode_params[1]#6.4455 Particles{Float64, 1} -# sol3flux222_p = solve(prob, alg) -# sol3flux222_p.estimated_ode_params[1]#6.40736 Particles{Float64, 1} -# sol3flux333_p = solve(prob, alg) -# sol3flux333_p.estimated_ode_params[1]#6.46214 Particles{Float64, 1} - -# # --------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, partial likelihood, dataset(0-1) -# # 25 points -# ta = range(tspan[1], 1.0, length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol0flux1_p = solve(prob, alg) -# sol0flux1_p.estimated_ode_params[1]#7.12625 Particles{Float64, 1} -# sol0flux2_p = solve(prob, alg) -# sol0flux2_p.estimated_ode_params[1]#8.40948 Particles{Float64, 1} -# sol0flux3_p = solve(prob, alg) -# sol0flux3_p.estimated_ode_params[1]#7.18768 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], 1.0, length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol0flux11_p = solve(prob, alg) -# sol0flux11_p.estimated_ode_params[1]#6.23707 Particles{Float64, 1} -# sol0flux22_p = solve(prob, alg) -# sol0flux22_p.estimated_ode_params[1]#6.09728 Particles{Float64, 1} -# sol0flux33_p = solve(prob, alg) -# sol0flux33_p.estimated_ode_params[1]#6.12971 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], 1.0, length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol0flux111_p = solve(prob, alg) -# sol0flux111_p.estimated_ode_params[1]#5.99039 Particles{Float64, 1} -# sol0flux222_p = solve(prob, alg) -# sol0flux222_p.estimated_ode_params[1]#5.89609 Particles{Float64, 1} -# sol0flux333_p = solve(prob, alg) -# sol0flux333_p.estimated_ode_params[1]#5.91923 Particles{Float64, 1} - -# # --------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood, dataset(1.0-2.0), Normal(12,5) distri prior -# # 25 points -# ta = range(1.0, tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 6.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f1 = solve(prob, alg) -# sol1f1.estimated_ode_params[1] -# # 10.9818Particles{Float64, 1} -# sol1f2 = solve(prob, alg) -# sol1f2.estimated_ode_params[1] -# # sol1f3 = solve(prob, alg) -# # sol1f3.estimated_ode_params[1] - -# # 50 points -# ta = range(1.0, tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 6.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f11 = solve(prob, alg) -# sol1f11.estimated_ode_params[1] -# sol1f22 = solve(prob, alg) -# sol1f22.estimated_ode_params[1] -# # sol1f33 = solve(prob, alg) -# # sol1f33.estimated_ode_params[1] - -# # 100 points -# ta = range(1.0, tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 6.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f111 = solve(prob, alg) -# sol1f111.estimated_ode_params[1] -# sol1f222 = solve(prob, alg) -# sol1f222.estimated_ode_params[1] -# # sol1f333 = solve(prob, alg) -# # sol1f333.estimated_ode_params[1] - -# # ---------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, partial likelihood, dataset(1.0-2.0), Normal(12,5) distri prior -# # 25 points -# ta = range(1.0, tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f1_p = solve(prob, alg) -# sol1f1_p.estimated_ode_params[1] -# sol1f2_p = solve(prob, alg) -# sol1f2_p.estimated_ode_params[1] -# sol1f3_p = solve(prob, alg) -# sol1f3_p.estimated_ode_params[1] - -# # 50 points -# ta = range(1.0, tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f11_p = solve(prob, alg) -# sol1f11_p.estimated_ode_params[1] -# sol1f22_p = solve(prob, alg) -# sol1f22_p.estimated_ode_params[1] -# sol1f33_p = solve(prob, alg) -# sol1f33_p.estimated_ode_params[1] - -# # 100 points -# ta = range(1.0, tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f111_p = solve(prob, alg) -# sol1f111_p.estimated_ode_params[1] -# sol1f222_p = solve(prob, alg) -# sol1f222_p.estimated_ode_params[1] -# sol1f333_p = solve(prob, alg) -# sol1f333_p.estimated_ode_params[1] - -# # ---------------------------------------------------------- - -# plot!(title = "9,2.5 50 training 2>full,1>partial") - -# p -# param1 -# # (lux chain) -# @prob mean(abs.(physsol2 .- sol3lux_pestim.ensemblesol[1])) < 8e-2 - -# # estimated parameters(lux chain) -# param1 = sol3lux_pestim.estimated_ode_params[1] -# @test abs(param1 - p) < abs(0.35 * p) - -# p -# param1 - -# # # my suggested Loss likelihood part -# # # + L2loss2(Tar, θ) -# # # My suggested extra loss function -# # function L2loss2(Tar::LogTargetDensity, θ) -# # f = Tar.prob.f - -# # # parameter estimation chosen or not -# # if Tar.extraparams > 0 -# # dataset = Tar.dataset - -# # # Timepoints to enforce Physics -# # dataset = Array(reduce(hcat, dataset)') -# # t = dataset[end, :] -# # û = dataset[1:(end - 1), :] - -# # ode_params = Tar.extraparams == 1 ? -# # θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : -# # θ[((length(θ) - Tar.extraparams) + 1):length(θ)] - -# # if length(û[:, 1]) == 1 -# # physsol = [f(û[:, i][1], -# # ode_params, -# # t[i]) -# # for i in 1:length(û[1, :])] -# # else -# # physsol = [f(û[:, i], -# # ode_params, -# # t[i]) -# # for i in 1:length(û[1, :])] -# # end -# # #form of NN output matrix output dim x n -# # deri_physsol = reduce(hcat, physsol) - -# # # OG deriv(basically gradient matching in case of an ODEFunction) -# # # in case of PDE or general ODE we would want to reduce residue of f(du,u,p,t) -# # # if length(û[:, 1]) == 1 -# # # deri_sol = [f(û[:, i][1], -# # # Tar.prob.p, -# # # t[i]) -# # # for i in 1:length(û[1, :])] -# # # else -# # # deri_sol = [f(û[:, i], -# # # Tar.prob.p, -# # # t[i]) -# # # for i in 1:length(û[1, :])] -# # # end -# # # deri_sol = reduce(hcat, deri_sol) -# # derivatives = calculate_derivatives(Tar.dataset) -# # deri_sol = reduce(hcat, derivatives) - -# # physlogprob = 0 -# # for i in 1:length(Tar.prob.u0) -# # # can add phystd[i] for u[i] -# # physlogprob += logpdf(MvNormal(deri_physsol[i, :], -# # LinearAlgebra.Diagonal(map(abs2, -# # Tar.l2std[i] .* -# # ones(length(deri_sol[i, :]))))), -# # deri_sol[i, :]) -# # end -# # return physlogprob -# # else -# # return 0 -# # end -# # end - -# # function calculate_derivatives(dataset) -# # x̂, time = dataset -# # num_points = length(x̂) - -# # # Initialize an array to store the derivative values. -# # derivatives = similar(x̂) - -# # for i in 2:(num_points - 1) -# # # Calculate the first-order derivative using central differences. -# # Δt_forward = time[i + 1] - time[i] -# # Δt_backward = time[i] - time[i - 1] - -# # derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) - -# # derivatives[i] = derivative -# # end - -# # # Derivatives at the endpoints can be calculated using forward or backward differences. -# # derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) -# # derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) - -# # return derivatives -# # end - -# size(dataset[1]) -# # Problem 1 with param estimation(flux,lux) -# # Normal -# # 6.20311 Particles{Float64, 1},6.21746Particles{Float64, 1} -# # better -# # 6.29093Particles{Float64, 1}, 6.27925Particles{Float64, 1} -# # Non ideal case -# # 6.14861Particles{Float64, 1}, -# sol2flux.estimated_ode_params -# sol2lux.estimated_ode_params[1] -# p -# size(sol3flux_pestim.ensemblesol[2]) -# plott = sol3flux_pestim.ensemblesol[1] -# using StatsPlots -# plotly() -# plot(t, sol3flux_pestim.ensemblesol[1]) - -# function calculate_derivatives(dataset) -# x̂, time = dataset -# num_points = length(x̂) - -# # Initialize an array to store the derivative values. -# derivatives = similar(x̂) - -# for i in 2:(num_points - 1) -# # Calculate the first-order derivative using central differences. -# Δt_forward = time[i + 1] - time[i] -# Δt_backward = time[i] - time[i - 1] - -# derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) - -# derivatives[i] = derivative -# end - -# # Derivatives at the endpoints can be calculated using forward or backward differences. -# derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) -# derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) - -# return derivatives -# end - -# # Example usage: -# # dataset = [x̂, time] -# derivatives = calculate_derivatives(dataset) -# dataset[1] -# # Access derivative values at specific time points as needed. - -# # # 9,0.5 -# # 0.09894916260292887 -# # 0.09870335436072103 -# # 0.08398556878067913 -# # 0.10109070099105527 -# # 0.09122683737517055 -# # 0.08614958011892977 -# # mean(abs.(x̂ .- meanscurve1)) #0.017112298305523976 -# # mean(abs.(physsol1 .- meanscurve1)) #0.004038636894341354 -# # # 9,4(little worse) -# # mean(abs.(x̂ .- meanscurve1))#0.01800876370000113 -# # mean(abs.(physsol1 .- meanscurve1))#0.007285681280600875 -# # # 30,30 -# # mean(abs.(x̂ .- meanscurve1)) #0.10599926120358046 -# # mean(abs.(physsol1 .- meanscurve1)) #0.10375554193397989 -# # # 30,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.10160824458252521 -# # mean(abs.(physsol1 .- meanscurve1)) #0.09999942538357891 - -# # # ------------------------------------------------normale -# # # 9,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.0333356493928835 -# # mean(abs.(physsol1 .- meanscurve1)) #0.02721733876400459 -# # # 9,4(little worse) -# # mean(abs.(x̂ .- meanscurve1)) #0.020734206709433347 -# # mean(abs.(physsol1 .- meanscurve1)) #0.012502850740700212 -# # # 30,30 -# # mean(abs.(x̂ .- meanscurve1)) #0.10615859683094729 -# # mean(abs.(physsol1 .- meanscurve1)) #0.10508141153722575 -# # # 30,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.10833514946031565 -# # mean(abs.(physsol1 .- meanscurve1)) #0.10668470203219232 - -# # # 9,0.5 -# # 10.158108285475553 -# # 10.207234384538026 -# # 10.215000657664852 -# # 10.213817644016174 -# # 13.380030074088719 -# # 13.348906350967326 - -# # 6.952731422892041 - -# # # All losses -# # 10.161478523326277 -# # # L2 losses 1 -# # 9.33312996960278 -# # # L2 losses 2 -# # 10.217417241370631 - -# # mean([fhsamples1[i][26] for i in 500:1000]) #6.245045767509431 -# # p #6.283185307179586 -# # # 9,4 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.212522300650451 -# # # 30,30 -# # mean([fhsamples1[i][23] for i in 500:1000]) #35.328636809737695 -# # # 30,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #35.232963812125654 - -# # # ---------------------------------------normale -# # # 9,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.547771572198114 -# # p #6.283185307179586 -# # # 9,4 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.158906185002702 -# # # 30,30 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.210400972620185 -# # # 30,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.153845019454522 - -# # # ----------------more dataset normale ----------------------------- -# # # 9,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.271141178216537 -# # p #6.283185307179586 -# # # 9,4 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.241144692919369 -# # # 30,30 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.124480447973127 -# # # 30,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.07838011629903 - -# # # 9,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.016551602015599295 -# # mean(abs.(physsol1 .- meanscurve1)) #0.0021488618484224245 -# # # 9,4(little worse) -# # mean(abs.(x̂ .- meanscurve1)) #0.017022725082640747 -# # mean(abs.(physsol1 .- meanscurve1)) #0.004339761917100232 -# # # 30,30 -# # mean(abs.(x̂ .- meanscurve1)) #0.09668785317864312 -# # mean(abs.(physsol1 .- meanscurve1)) #0.09430712337543362 -# # # 30,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.09958118358974392 -# # mean(abs.(physsol1 .- meanscurve1)) #0.09717454226368502 - -# # # ----------------more dataset special ----------------------------- -# # # 9,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.284355334485365 -# # p #6.283185307179586 -# # # 9,4 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.259238106698602 -# # # 30,30 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.139808934336987 -# # # 30,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.03921327641226 - -# # # 9,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.016627231605546876 -# # mean(abs.(physsol1 .- meanscurve1)) #0.0020311429130039564 -# # # 9,4(little worse) -# # mean(abs.(x̂ .- meanscurve1)) #0.016650324577507352 -# # mean(abs.(physsol1 .- meanscurve1)) #0.0027537543411154677 -# # # 30,30 -# # mean(abs.(x̂ .- meanscurve1)) #0.09713187937270151 -# # mean(abs.(physsol1 .- meanscurve1)) #0.09317278450371556 -# # # 30,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.09550234866855814 -# # mean(abs.(physsol1 .- meanscurve1)) #0.09317278450371556 - -# # using Plots, StatsPlots -# # plotly() - -# # --------------------------------------------------------- -# # # # Distribution abstract in wrapper, dataset Float64 -# # # 268.651 s (206393690 allocations: 388.71 GiB) -# # # 318.170551 seconds (206.29 M allocations: 388.453 GiB, 20.83% gc time) - -# # # # Above with dataset Real subtype -# # # 326.201 s (206327409 allocations: 388.42 GiB) -# # # 363.189370 seconds (206.25 M allocations: 387.975 GiB, 15.77% gc time) -# # # 306.171 s (206321277 allocations: 388.55 GiB) -# # # 356.180699 seconds (206.43 M allocations: 388.361 GiB, 13.77% gc time) - -# # # # Above with dataset AbstractFloat subtype -# # # 290.751187 seconds (205.94 M allocations: 387.955 GiB, 12.92% gc time) -# # # 296.319815 seconds (206.38 M allocations: 388.730 GiB, 12.69% gc time) - -# # # # ODEProblem float64 dtaset and vector distri inside -# # # 273.169 s (206128318 allocations: 388.40 GiB) -# # # 274.059531 seconds (205.91 M allocations: 387.953 GiB, 12.77% gc time) - -# # # # Dataset float64 inside and vector distri outsude -# # # 333.603 s (206251143 allocations: 388.41 GiB) -# # # 373.377222 seconds (206.11 M allocations: 387.968 GiB, 13.25% gc time) -# # # 359.745 s (206348301 allocations: 388.41 GiB) -# # # 357.813114 seconds (206.31 M allocations: 388.354 GiB, 13.54% gc time) - -# # # # Dataset float64 inside and vector distri inside -# # # 326.437 s (206253571 allocations: 388.41 GiB) -# # # 290.334083 seconds (205.92 M allocations: 387.954 GiB, 13.82% gc time) - -# # # # current setting -# # # 451.304 s (206476927 allocations: 388.43 GiB) -# # # 384.532732 seconds (206.22 M allocations: 387.976 GiB, 13.17% gc time) -# # # 310.223 s (206332558 allocations: 388.63 GiB) -# # # 344.243889 seconds (206.34 M allocations: 388.409 GiB, 13.84% gc time) -# # # 357.457737 seconds (206.66 M allocations: 389.064 GiB, 18.16% gc time) - -# # # # shit setup -# # # 325.595 s (206283732 allocations: 388.41 GiB) -# # # 334.248753 seconds (206.06 M allocations: 387.964 GiB, 12.60% gc time) -# # # 326.011 s (206370857 allocations: 388.56 GiB) -# # # 327.203339 seconds (206.29 M allocations: 388.405 GiB, 12.92% gc time) - -# # # # in wrapper Distribution prior, insiade FLOAT64 DATASET -# # # 325.158167 seconds (205.97 M allocations: 387.958 GiB, 15.07% gc time) -# # # 429.536 s (206476324 allocations: 388.43 GiB) -# # # 527.364 s (206740343 allocations: 388.58 GiB) - -# # # # wrapper Distribtuion, inside Float64 -# # # 326.017 s (206037971 allocations: 387.96 GiB) -# # # 347.424730 seconds (206.45 M allocations: 388.532 GiB, 12.92% gc time) - -# # # 439.047568 seconds (284.24 M allocations: 392.598 GiB, 15.25% gc time, 14.36% compilation time: 0% of which was recompilation) -# # # 375.472142 seconds (206.40 M allocations: 388.529 GiB, 14.93% gc time) -# # # 374.888820 seconds (206.34 M allocations: 388.346 GiB, 14.09% gc time) -# # # 363.719611 seconds (206.39 M allocations: 388.581 GiB, 15.08% gc time) -# # # # inside Distribtion, instide Float64 -# # # 310.238 s (206324249 allocations: 388.53 GiB) -# # # 308.991494 seconds (206.34 M allocations: 388.549 GiB, 14.01% gc time) -# # # 337.442 s (206280712 allocations: 388.36 GiB) -# # # 299.983096 seconds (206.29 M allocations: 388.512 GiB, 17.14% gc time) - -# # # 394.924357 seconds (206.27 M allocations: 388.337 GiB, 23.68% gc time) -# # # 438.204179 seconds (206.39 M allocations: 388.470 GiB, 23.84% gc time) -# # # 376.626914 seconds (206.46 M allocations: 388.693 GiB, 18.72% gc time) -# # # 286.863795 seconds (206.14 M allocations: 388.370 GiB, 18.80% gc time) -# # # 285.556929 seconds (206.22 M allocations: 388.371 GiB, 17.04% gc time) -# # # 291.471662 seconds (205.96 M allocations: 388.068 GiB, 19.85% gc time) - -# # # 495.814341 seconds (284.62 M allocations: 392.622 GiB, 12.56% gc time, 10.96% compilation time: 0% of which was recompilation) -# # # 361.530617 seconds (206.36 M allocations: 388.526 GiB, 14.98% gc time) -# # # 348.576065 seconds (206.22 M allocations: 388.337 GiB, 15.01% gc time) -# # # 374.575609 seconds (206.45 M allocations: 388.586 GiB, 14.65% gc time) -# # # 314.223008 seconds (206.23 M allocations: 388.411 GiB, 14.63% gc time) - -# # PROBLEM-3 LOTKA VOLTERRA EXAMPLE [WIP] (WITH PARAMETER ESTIMATION)(will be put in tutorial page) -# function lotka_volterra(u, p, t) -# # Model parameters. -# α, β, γ, δ = p -# # Current state. -# x, y = u - -# # Evaluate differential equations. -# dx = (α - β * y) * x # prey -# dy = (δ * x - γ) * y # predator - -# return [dx, dy] -# end - -# u0 = [1.0, 1.0] -# p = [1.5, 1.0, 3.0, 1.0] -# tspan = (0.0, 6.0) -# prob = ODEProblem(lotka_volterra, u0, tspan, p) -# solution = solve(prob, Tsit5(); saveat = 0.05) - -# as = reduce(hcat, solution.u) -# as[1, :] -# # Plot simulation. -# time = solution.t -# u = hcat(solution.u...) -# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct -# x = u[1, :] + 0.5 * randn(length(u[1, :])) -# y = u[2, :] + 0.5 * randn(length(u[1, :])) -# dataset = [x[1:50], y[1:50], time[1:50]] -# # scatter!(time, [x, y]) -# # scatter!(dataset[3], [dataset[2], dataset[1]]) - -# # NN has 2 outputs as u -> [dx,dy] -# chainlux1 = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), -# Lux.Dense(6, 2)) -# chainflux1 = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) - -# # fh_mcmc_chainflux1, fhsamplesflux1, fhstatsflux1 = ahmc_bayesian_pinn_ode(prob, chainflux1, -# # dataset = dataset, -# # draw_samples = 1000, -# # l2std = [ -# # 0.05, -# # 0.05, -# # ], -# # phystd = [ -# # 0.05, -# # 0.05, -# # ], -# # priorsNNw = (0.0, -# # - -# # 3.0)) - -# # check if NN output is more than 1 -# # numoutput = size(luxar[1])[1] -# # if numoutput > 1 -# # # Initialize a vector to store the separated outputs for each output dimension -# # output_matrices = [Vector{Vector{Float32}}() for _ in 1:numoutput] - -# # # Loop through each element in the `as` vector -# # for element in as -# # for i in 1:numoutput -# # push!(output_matrices[i], element[i, :]) # Append the i-th output (i-th row) to the i-th output_matrices -# # end -# # end - -# # ensemblecurves = Vector{}[] -# # for r in 1:numoutput -# # br = hcat(output_matrices[r]...)' -# # ensemblecurve = prob.u0[r] .+ -# # [Particles(br[:, i]) for i in 1:length(t)] .* -# # (t .- prob.tspan[1]) -# # push!(ensemblecurves, ensemblecurve) -# # end - -# # else -# # # ensemblecurve = prob.u0 .+ -# # # [Particles(reduce(vcat, luxar)[:, i]) for i in 1:length(t)] .* -# # # (t .- prob.tspan[1]) -# # print("yuh") -# # end - -# # fhsamplesflux2 -# # nnparams = length(init1) -# # estimnnparams = [Particles(reduce(hcat, fhsamplesflux2)[i, :]) for i in 1:nnparams] -# # ninv=4 -# # estimated_params = [Particles(reduce(hcat, fhsamplesflux2[(end - ninv + 1):end])[i, :]) -# # for i in (nnparams + 1):(nnparams + ninv)] -# # output_matrices[r] -# # br = hcat(output_matrices[r]...)' - -# # br[:, 1] - -# # [Particles(br[:, i]) for i in 1:length(t)] -# # prob.u0 -# # [Particles(br[:, i]) for i in 1:length(t)] .* -# # (t .- prob.tspan[1]) - -# # ensemblecurve = prob.u0[r] .+ -# # [Particles(br[:, i]) for i in 1:length(t)] .* -# # (t .- prob.tspan[1]) -# # push!(ensemblecurves, ensemblecurve) - -# using StatsPlots -# plotly() -# plot(t, ensemblecurve) -# plot(t, ensemblecurves[1]) -# plot!(t, ensemblecurves[2]) -# ensemblecurve -# ensemblecurves[1] -# fh_mcmc_chainflux2, fhsamplesflux2, fhstatsflux2 = ahmc_bayesian_pinn_ode(prob, chainflux1, -# dataset = dataset, -# draw_samples = 1000, -# l2std = [ -# 0.05, -# 0.05, -# ], -# phystd = [ -# 0.05, -# 0.05, -# ], -# priorsNNw = (0.0, -# 3.0), -# param = [ -# Normal(1.5, -# 0.5), -# Normal(1.2, -# 0.5), -# Normal(3.3, -# 0.5), -# Normal(1.4, -# 0.5), -# ], progress = true) - -# alg = NeuralPDE.BNNODE(chainflux1, -# dataset = dataset, -# draw_samples = 1000, -# l2std = [ -# 0.05, -# 0.05, -# ], -# phystd = [ -# 0.05, -# 0.05, -# ], -# priorsNNw = (0.0, -# 3.0), -# param = [ -# Normal(4.5, -# 5), -# Normal(7, -# 2), -# Normal(5, -# 2), -# Normal(-4, -# 6), -# ], -# n_leapfrog = 30, progress = true) - -# sol3flux_pestim = solve(prob, alg) - -# # OG PARAM VALUES -# [1.5, 1.0, 3.0, 1.0] -# # less -# # [1.34, 7.51, 2.54, -2.55] -# # better -# # [1.48, 0.993, 2.77, 0.954] - -# sol3flux_pestim.es -# sol3flux_pestim.estimated_ode_params -# # fh_mcmc_chainlux1, fhsampleslux1, fhstatslux1 = ahmc_bayesian_pinn_ode(prob, chainlux1, -# # dataset = dataset, -# # draw_samples = 1000, -# # l2std = [0.05, 0.05], -# # phystd = [ -# # 0.05, -# # 0.05, -# # ], -# # priorsNNw = (0.0, -# # 3.0)) - -# # fh_mcmc_chainlux2, fhsampleslux2, fhstatslux2 = ahmc_bayesian_pinn_ode(prob, chainlux1, -# # dataset = dataset, -# # draw_samples = 1000, -# # l2std = [0.05, 0.05], -# # phystd = [ -# # 0.05, -# # 0.05, -# # ], -# # priorsNNw = (0.0, -# # 3.0), -# # param = [ -# # Normal(1.5, 0.5), -# # Normal(1.2, 0.5), -# # Normal(3.3, 0.5), -# # Normal(1.4, 0.5), -# # ]) - -# init1, re1 = destructure(chainflux1) -# θinit, st = Lux.setup(Random.default_rng(), chainlux1) -# # PLOT testing points -# t = time -# p = prob.p -# collect(Float64, vcat(ComponentArrays.ComponentArray(θinit))) -# collect(Float64, ComponentArrays.ComponentArray(θinit)) -# # Mean of last 1000 sampled parameter's curves(flux and lux chains)[Ensemble predictions] -# out = re1.([fhsamplesflux1[i][1:68] for i in 500:1000]) -# yu = [out[i](t') for i in eachindex(out)] - -# function getensemble(yu, num_models) -# num_rows, num_cols = size(yu[1]) -# row_means = zeros(Float32, num_rows, num_cols) -# for i in 1:num_models -# row_means .+= yu[i] -# end -# row_means ./ num_models -# end -# fluxmean = getensemble(yu, length(out)) -# meanscurve1_1 = prob.u0 .+ (t' .- prob.tspan[1]) .* fluxmean -# mean(abs.(u .- meanscurve1_1)) - -# plot!(t, physsol1) -# @test mean(abs2.(x̂ .- meanscurve1_1)) < 2e-2 -# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 -# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 -# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 - -# out = re1.([fhsamplesflux2[i][1:68] for i in 500:1000]) -# yu = collect(out[i](t') for i in eachindex(out)) -# fluxmean = getensemble(yu, length(out)) -# meanscurve1_2 = prob.u0 .+ (t' .- prob.tspan[1]) .* fluxmean -# mean(abs.(u .- meanscurve1_2)) - -# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 -# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 -# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 -# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 - -# θ = [vector_to_parameters(fhsampleslux1[i][1:(end - 4)], θinit) for i in 500:1000] -# luxar = [chainlux1(t', θ[i], st)[1] for i in 1:500] -# luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] -# meanscurve2_1 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean - -# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 -# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 -# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 -# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 - -# θ = [vector_to_parameters(fhsampleslux2[i][1:(end - 4)], θinit) for i in 500:1000] -# luxar = [chainlux1(t', θ[i], st)[1] for i in 1:500] -# luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] -# meanscurve2_2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean - -# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 -# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 -# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 -# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 - -# # # ESTIMATED ODE PARAMETERS (NN1 AND NN2) -# @test abs(p - mean([fhsamplesflux2[i][69] for i in 500:1000])) < 0.1 * p[1] -# @test abs(p - mean([fhsampleslux2[i][69] for i in 500:1000])) < 0.2 * p[1] - -# # @test abs(p - mean([fhsamplesflux2[i][70] for i in 500:1000])) < 0.1 * p[2] -# # @test abs(p - mean([fhsampleslux2[i][70] for i in 500:1000])) < 0.2 * p[2] - -# # @test abs(p - mean([fhsamplesflux2[i][71] for i in 500:1000])) < 0.1 * p[3] -# # @test abs(p - mean([fhsampleslux2[i][71] for i in 500:1000])) < 0.2 * p[3] - -# # @test abs(p - mean([fhsamplesflux2[i][72] for i in 500:1000])) < 0.1 * p[4] -# # @test abs(p - mean([fhsampleslux2[i][72] for i in 500:1000])) < 0.2 * p[4] - -# # fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainlux1, -# # dataset = dataset, -# # draw_samples = 1000, -# # l2std = [0.05, 0.05], -# # phystd = [0.05, 0.05], -# # priorsNNw = (0.0, 3.0), -# # param = [ -# # Normal(1.5, 0.5), -# # Normal(1.2, 0.5), -# # Normal(3.3, 0.5), -# # Normal(1.4, 0.5), -# # ], autodiff = true) - -# # fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainlux1, -# # dataset = dataset, -# # draw_samples = 1000, -# # l2std = [0.05, 0.05], -# # phystd = [0.05, 0.05], -# # priorsNNw = (0.0, 3.0), -# # param = [ -# # Normal(1.5, 0.5), -# # Normal(1.2, 0.5), -# # Normal(3.3, 0.5), -# # Normal(1.4, 0.5), -# # ], nchains = 2) - -# # NOTES (WILL CLEAR LATER) -# # -------------------------------------------------------------------------------------------- -# # Hamiltonian energy must be lowest(more paramters the better is it to map onto them) -# # full better than L2 and phy individual(test) -# # in mergephys more points after training points is better from 20->40 -# # does consecutive runs bceome better? why?(plot 172)(same chain maybe) -# # does density of points in timespan matter dataset vs internal timespan?(plot 172)(100+0.01) -# # when training from 0-1 and phys from 1-5 with 1/150 simple nn slow,but bigger nn faster decrease in Hmailtonian -# # bigger time interval more curves to adapt to only more parameters adapt to that, better NN architecture -# # higher order logproblems solve better -# # repl up up are same instances? but reexecute calls are new? - -# #Compare results against paper example -# # Lux chains support (DONE) -# # fix predictions for odes depending upon 1,p in f(u,p,t)(DONE) -# # lotka volterra learn curve beyond l2 losses(L2 losses determine accuracy of parameters)(parameters cant run free ∴ L2 interval only) -# # check if prameters estimation works(YES) -# # lotka volterra parameters estimate (DONE) - -# using NeuralPDE, Lux, Flux, Optimization, OptimizationOptimJL -# import ModelingToolkit: Interval -# using Plots, StatsPlots -# plotly() -# # Profile.init() - -# @parameters x y -# @variables u(..) -# Dxx = Differential(x)^2 -# Dyy = Differential(y)^2 - -# # 2D PDE -# eq = Dxx(u(x, y)) + Dyy(u(x, y)) ~ -sin(pi * x) * sin(pi * y) - -# # Boundary conditions -# bcs = [u(0, y) ~ 0.0, u(1, y) ~ 0.0, -# u(x, 0) ~ 0.0, u(x, 1) ~ 0.0] -# # Space and time domains -# domains = [x ∈ Interval(0.0, 1.0), -# y ∈ Interval(0.0, 1.0)] - -# # Neural network -# dim = 2 # number of dimensions -# chain = Flux.Chain(Flux.Dense(dim, 16, Lux.σ), Flux.Dense(16, 16, Lux.σ), Flux.Dense(16, 1)) -# θ, re = destructure(chain) -# # Discretization -# dx = 0.05 -# discretization = PhysicsInformedNN(chain, GridTraining(dx)) - -# @named pde_system = PDESystem(eq, bcs, domains, [x, y], [u(x, y)]) - -# pinnrep = symbolic_discretize(pde_system, discretization) -# typeof(pinnrep.phi) -# typeof(pinnrep.phi) -# typeof(re) -# pinnrep.phi([1, 2], θ) - -# typeof(θ) - -# print(pinnrep) -# pinnrep.eqs -# pinnrep.bcs -# pinnrep.domains -# pinnrep.eq_params -# pinnrep.defaults -# print(pinnrep.default_p) -# pinnrep.param_estim -# print(pinnrep.additional_loss) -# pinnrep.adaloss -# pinnrep.depvars -# pinnrep.indvars -# pinnrep.dict_depvar_input -# pinnrep.dict_depvars -# pinnrep.dict_indvars -# print(pinnrep.logger) -# pinnrep.multioutput -# pinnrep.iteration -# pinnrep.init_params -# pinnrep.flat_init_params -# pinnrep.phi -# pinnrep.derivative -# pinnrep.strategy -# pinnrep.pde_indvars -# pinnrep.bc_indvars -# pinnrep.pde_integration_vars -# pinnrep.bc_integration_vars -# pinnrep.integral -# pinnrep.symbolic_pde_loss_functions -# pinnrep.symbolic_bc_loss_functions -# pinnrep.loss_functions - -# # = discretize(pde_system, discretization) -# prob = symbolic_discretize(pde_system, discretization) -# # "The boundary condition loss functions" -# sum([prob.loss_functions.bc_loss_functions[i](θ) for i in eachindex(1:4)]) -# sum([prob.loss_functions.pde_loss_functions[i](θ) for i in eachindex(1)]) - -# prob.loss_functions.full_loss_function(θ, 32) - -# prob.loss_functions.bc_loss_functions[1](θ) - -# prob.loss_functions.bc_loss_functions -# prob.loss_functions.full_loss_function -# prob.loss_functions.additional_loss_function -# prob.loss_functions.pde_loss_functions - -# 1.3953060473003345 + 1.378102161087438 + 1.395376727128639 + 1.3783868705075002 + -# 0.22674532775196876 -# # "The PDE loss functions" -# prob.loss_functions.pde_loss_functions -# prob.loss_functions.pde_loss_functions[1](θ) -# # "The full loss function, combining the PDE and boundary condition loss functions.This is the loss function that is used by the optimizer." -# prob.loss_functions.full_loss_function(θ, nothing) -# prob.loss_functions.full_loss_function(θ, 423423) - -# # "The wrapped `additional_loss`, as pieced together for the optimizer." -# prob.loss_functions.additional_loss_function -# # "The pre-data version of the PDE loss function" -# prob.loss_functions.datafree_pde_loss_functions -# # "The pre-data version of the BC loss function" -# prob.loss_functions.datafree_bc_loss_functions - -# using Random -# θ, st = Lux.setup(Random.default_rng(), chain) -# #Optimizer -# opt = OptimizationOptimJL.BFGS() - -# #Callback function -# callback = function (p, l) -# println("Current loss is: $l") -# return false -# end - -# res = Optimization.solve(prob, opt, callback = callback, maxiters = 1000) -# phi = discretization.phi - -# # ------------------------------------------------ -# using NeuralPDE, Lux, ModelingToolkit, Optimization, OptimizationOptimJL, OrdinaryDiffEq, -# Plots -# import ModelingToolkit: Interval, infimum, supremum -# @parameters t, σ_, β, ρ -# @variables x(..), y(..), z(..) -# Dt = Differential(t) -# eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), -# Dt(y(t)) ~ x(t) * (ρ - z(t)) - y(t), -# Dt(z(t)) ~ x(t) * y(t) - β * z(t)] - -# bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] -# domains = [t ∈ Interval(0.0, 1.0)] -# dt = 0.01 - -# input_ = length(domains) -# n = 8 -# chain1 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, 1)) -# chain2 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, 1)) -# chain3 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, 1)) - -# function lorenz!(du, u, p, t) -# du[1] = 10.0 * (u[2] - u[1]) -# du[2] = u[1] * (28.0 - u[3]) - u[2] -# du[3] = u[1] * u[2] - (8 / 3) * u[3] -# end - -# u0 = [1.0; 0.0; 0.0] -# tspan = (0.0, 1.0) -# prob = ODEProblem(lorenz!, u0, tspan) -# sol = solve(prob, Tsit5(), dt = 0.1) -# ts = [infimum(d.domain):dt:supremum(d.domain) for d in domains][1] -# function getData(sol) -# data = [] -# us = hcat(sol(ts).u...) -# ts_ = hcat(sol(ts).t...) -# return [us, ts_] -# end -# data = getData(sol) - -# (u_, t_) = data -# len = length(data[2]) - -# depvars = [:x, :y, :z] -# function additional_loss(phi, θ, p) -# return sum(sum(abs2, phi[i](t_, θ[depvars[i]]) .- u_[[i], :]) / len for i in 1:1:3) -# end - -# discretization = NeuralPDE.PhysicsInformedNN([chain1, chain2, chain3], -# NeuralPDE.GridTraining(dt), -# param_estim = false, -# additional_loss = additional_loss) -# @named pde_system = PDESystem(eqs, bcs, domains, [t], [x(t), y(t), z(t)], [σ_, ρ, β], -# defaults = Dict([p .=> 1.0 for p in [σ_, ρ, β]])) -# prob = NeuralPDE.discretize(pde_system, discretization) -# callback = function (p, l) -# println("Current loss is: $l") -# return false -# end -# res = Optimization.solve(prob, BFGS(); callback = callback, maxiters = 5000) -# p_ = res.u[(end - 2):end] # p_ = [9.93, 28.002, 2.667] - -# minimizers = [res.u.depvar[depvars[i]] for i in 1:3] -# ts = [infimum(d.domain):(dt / 10):supremum(d.domain) for d in domains][1] -# u_predict = [[discretization.phi[i]([t], minimizers[i])[1] for t in ts] for i in 1:3] -# plot(sol) -# plot!(ts, u_predict, label = ["x(t)" "y(t)" "z(t)"]) - -# discretization.multioutput -# discretization.chain -# discretization.strategy -# discretization.init_params -# discretization.phi -# discretization.derivative -# discretization.param_estim -# discretization.additional_loss -# discretization.adaptive_loss -# discretization.logger -# discretization.log_options -# discretization.iteration -# discretization.self_increment -# discretization.multioutput -# discretization.kwargs - -# struct BNNODE1{P <: Vector{<:Distribution}} -# chain::Any -# Kernel::Any -# draw_samples::UInt32 -# priorsNNw::Tuple{Float64, Float64} -# param::P -# l2std::Vector{Float64} -# phystd::Vector{Float64} - -# function BNNODE1(chain, Kernel; draw_samples = 2000, priorsNNw = (0.0, 3.0), param = [], -# l2std = [0.05], phystd = [0.05]) -# BNNODE1(chain, Kernel, draw_samples, priorsNNw, param, l2std, phystd) -# end -# end - -# struct BNNODE3{C, K, P <: Union{Any, Vector{<:Distribution}}} -# chain::C -# Kernel::K -# draw_samples::UInt32 -# priorsNNw::Tuple{Float64, Float64} -# param::P -# l2std::Vector{Float64} -# phystd::Vector{Float64} - -# function BNNODE3(chain, Kernel; draw_samples = 2000, priorsNNw = (0.0, 3.0), param = [], -# l2std = [0.05], phystd = [0.05]) -# new{typeof(chain), typeof(Kernel), typeof(param)}(chain, Kernel, draw_samples, -# priorsNNw, param, l2std, phystd) -# end -# end -# linear_analytic = (u0, p, t) -> u0 + sin(2 * π * t) / (2 * π) -# linear = (u, p, t) -> cos(2 * π * t) -# tspan = (0.0, 2.0) -# u0 = 0.0 -# prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan) - -# ta = range(tspan[1], tspan[2], length = 300) -# u = [linear_analytic(u0, nothing, ti) for ti in ta] -# sol1 = solve(prob, Tsit5()) - -# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct -# x̂ = collect(Float64, Array(u) + 0.02 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂[1:100], time[1:100]] - -# # Call BPINN, create chain -# chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) -# chainlux = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) -# HMC -# solve(prob, BNNODE(chainflux, HMC)) -# BNNODE1(chainflux, HMC, 2000) - -# draw_samples = 2000 -# priorsNNw = (0.0, 3.0) -# param = [] -# l2std = [0.05] -# phystd = [0.05] -# @time BNNODE3(chainflux, HMC, draw_samples = 2000, priorsNNw = (0.0, 3.0), -# param = [nothing], -# l2std = [0.05], phystd = [0.05]) -# typeof(Nothing) <: Vector{<:Distribution} -# Nothing <: Distribution -# {UnionAll} <: Distribution -# @time [Nothing] -# typeof([Nothing]) -# @time [1] - -# function test1(sum; c = 23, d = 32) -# return sum + c + d -# end -# function test(a, b; c, d) -# return test1(a + b, c, d) -# end - -# test(2, 2) - -# struct BNNODE3{C, K, P <: Union{Vector{Nothing}, Vector{<:Distribution}}} -# chain::C -# Kernel::K -# draw_samples::Int64 -# priorsNNw::Tuple{Float64, Float64} -# param::P -# l2std::Vector{Float64} -# phystd::Vector{Float64} - -# function BNNODE3(chain, Kernel; draw_samples, -# priorsNNw, param = [nothing], l2std, phystd) -# new{typeof(chain), typeof(Kernel), typeof(param)}(chain, -# Kernel, -# draw_samples, -# priorsNNw, -# param, l2std, -# phystd) -# end -# end - -# function solve1(prob::DiffEqBase.AbstractODEProblem, alg::BNNODE3; -# dataset = [nothing], dt = 1 / 20.0, -# init_params = nothing, nchains = 1, -# autodiff = false, Integrator = Leapfrog, -# Adaptor = StanHMCAdaptor, targetacceptancerate = 0.8, -# Metric = DiagEuclideanMetric, jitter_rate = 3.0, -# tempering_rate = 3.0, max_depth = 10, Δ_max = 1000, -# n_leapfrog = 10, δ = 0.65, λ = 0.3, progress = true, -# verbose = false) -# chain = alg.chain -# l2std = alg.l2std -# phystd = alg.phystd -# priorsNNw = alg.priorsNNw -# Kernel = alg.Kernel -# draw_samples = alg.draw_samples - -# param = alg.param == [nothing] ? [] : alg.param -# mcmcchain, samples, statistics = ahmc_bayesian_pinn_ode(prob, chain, dataset = dataset, -# draw_samples = draw_samples, -# init_params = init_params, -# physdt = dt, l2std = l2std, -# phystd = phystd, -# priorsNNw = priorsNNw, -# param = param, -# nchains = nchains, -# autodiff = autodiff, -# Kernel = Kernel, -# Integrator = Integrator, -# Adaptor = Adaptor, -# targetacceptancerate = targetacceptancerate, -# Metric = Metric, -# jitter_rate = jitter_rate, -# tempering_rate = tempering_rate, -# max_depth = max_depth, -# Δ_max = Δ_max, -# n_leapfrog = n_leapfrog, δ = δ, -# λ = λ, progress = progress, -# verbose = verbose) -# end - -# linear_analytic = (u0, p, t) -> u0 + sin(2 * π * t) / (2 * π) -# linear = (u, p, t) -> cos(2 * π * t) -# tspan = (0.0, 2.0) -# u0 = 0.0 -# prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan) - -# ta = range(tspan[1], tspan[2], length = 300) -# u = [linear_analytic(u0, nothing, ti) for ti in ta] -# # sol1 = solve(prob, Tsit5()) - -# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct -# x̂ = collect(Float64, Array(u) + 0.02 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂[1:100], time[1:100]] - -# # Call BPINN, create chain -# chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) -# chainlux = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) -# HMC - -# solve1(prob, a) -# a = BNNODE3(chainflux, HMC, draw_samples = 2000, -# priorsNNw = (0.0, 3.0), -# l2std = [0.05], phystd = [0.05]) - -# Define Lotka-Volterra model. -function lotka_volterra1(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end - -u0 = [1.0, 1.0] -p = [1.5, 1.0, 3.0, 1.0] -tspan = (0.0, 6.0) -prob = ODEProblem(lotka_volterra1, u0, tspan, p) -solution = solve(prob, Tsit5(); saveat = 0.05) - -as = reduce(hcat, solution.u) -as[1, :] -# Plot simulation. -time = solution.t -u = hcat(solution.u...) -# BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct -x = u[1, :] + 0.5 * randn(length(u[1, :])) -y = u[2, :] + 0.5 * randn(length(u[1, :])) -dataset = [x[1:50], y[1:50], time[1:50]] -# scatter!(time, [x, y]) -# scatter!(dataset[3], [dataset[2], dataset[1]]) - -# NN has 2 outputs as u -> [dx,dy] -chainlux1 = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), - Lux.Dense(6, 2)) -chainflux1 = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) - -fh_mcmc_chainflux1, fhsamplesflux1, fhstatsflux1 = ahmc_bayesian_pinn_ode(prob, chainflux1, - dataset = dataset, - draw_samples = 1000, - l2std = [ - 0.05, - 0.05, - ], - phystd = [ - 0.05, - 0.05, - ], - priorsNNw = (0.0, 3.0), progress = true) -ahmc_bayesian_pinn_ode(prob, chainflux1, - dataset = dataset, - draw_samples = 1000, - l2std = [ - 0.05, - 0.05, - ], - phystd = [ - 0.05, - 0.05, - ], - priorsNNw = (0.0, 3.0), progress = true) - -# 2×171 Matrix{Float64}: -# -0.5 -0.518956 -0.529639 … -1.00266 -1.01049 -# 2.0 1.97109 1.92747 0.42619 0.396335 - -# 2-element Vector{Float64}: -# -119451.94949911036 -# -128543.23714618056 - -# alg = NeuralPDE.BNNODE(chainflux1, -# dataset = dataset, -# draw_samples = 1000, -# l2std = [ -# 0.05, -# 0.05, -# ], -# phystd = [ -# 0.05, -# 0.05, -# ], -# priorsNNw = (0.0, -# 3.0), -# param = [ -# Normal(4.5, -# 5), -# Normal(7, -# 2), -# Normal(5, -# 2), -# Normal(-4, -# 6), -# ], -# n_leapfrog = 30, progress = true) - -# sol3flux_pestim = solve(prob, alg) - -# ---------------------------------------------- -# original paper implementation -# 25 points -run1 #7.70593 Particles{Float64, 1} -run2 #6.66347 Particles{Float64, 1} -run3 #6.84827 Particles{Float64, 1} - -# 50 points -run1 #7.83577 Particles{Float64, 1} -run2 #6.49477 Particles{Float64, 1} -run3 #6.47421 Particles{Float64, 1} - -# 100 points -run1 #5.96604 Particles{Float64, 1} -run2 #6.05432 Particles{Float64, 1} -run3 #6.08856 Particles{Float64, 1} - -# Full likelihood(uses total variation regularized differentiation) -# 25 points -run1 #6.41722 Particles{Float64, 1} -run2 #6.42782 Particles{Float64, 1} -run3 #6.42782 Particles{Float64, 1} - -# 50 points -run1 #5.71268 Particles{Float64, 1} -run2 #5.74599 Particles{Float64, 1} -run3 #5.74599 Particles{Float64, 1} - -# 100 points -run1 #6.59097 Particles{Float64, 1} -run2 #6.62813 Particles{Float64, 1} -run3 #6.62813 Particles{Float64, 1} - -using Plots, StatsPlots -function lotka_volterra(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end - -# initial-value problem. -u0 = [1.0, 1.0] -p = [1.5, 1.0, 3.0, 1.0] -tspan = (0.0, 6.0) -prob = ODEProblem(lotka_volterra, u0, tspan, p) - -# Plot simulation. - -solution = solve(prob, Tsit5(); saveat = 0.05) -plot(solve(prob, Tsit5())) - -# Dataset creation for parameter estimation -time = solution.t -u = hcat(solution.u...) -x = u[1, :] + 0.5 * randn(length(u[1, :])) -y = u[2, :] + 0.5 * randn(length(u[1, :])) -dataset = [x, y, time] - -# Neural Networks must have 2 outputs as u -> [dx,dy] in function lotka_volterra() -chainflux = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) |> - Flux.f64 - -chainlux = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), Lux.Dense(6, 2)) - -alg1 = NeuralPDE.BNNODE(chainflux, - dataset = dataset, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol_flux_pestim = solve(prob, alg1) - -# Dataset not needed as we are solving the equation with ideal parameters -alg2 = NeuralPDE.BNNODE(chainlux, - draw_samples = 1000, - l2std = [ - 0.05, - 0.05, - ], - phystd = [ - 0.05, - 0.05, - ], - priorsNNw = (0.0, - 3.0), - n_leapfrog = 30, progress = true) - -sol_lux = solve(prob, alg2) - -#testing timepoints must match keyword arg `saveat`` timepoints of solve() call -t = collect(Float64, prob.tspan[1]:(1 / 50.0):prob.tspan[2]) - -# plotting solution for x,y for chain_flux -plot(t, sol_flux_pestim.ensemblesol[1]) -plot!(t, sol_flux_pestim.ensemblesol[2]) - -plot(sol_flux_pestim.ens1mblesol[1]) -plot!(sol_flux_pestim.ensemblesol[2]) - -# estimated ODE parameters by .estimated_ode_params, weights and biases by .estimated_nn_params -sol_flux_pestim.estimated_nn_params -sol_flux_pestim.estimated_ode_params - -# plotting solution for x,y for chain_lux -plot(t, sol_lux.ensemblesol[1]) -plot!(t, sol_lux.ensemblesol[2]) - -# estimated weights and biases by .estimated_nn_params for chain_lux -sol_lux.estimated_nn_params - -# # ----------------------------------stats----------------------------- -# # ---------------------------- -# # ----------------------------- -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:04:47 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:03:38 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:04:12 -# # -------------------------- -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:05:09 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:04:47 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:04:25 -# # -------------- -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:06:47 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:05:54 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:05:46 -# # ------------------------ -# # ----------------------- -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -882.2934218498742 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:04:06 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -882.2934218498742 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:03:32 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -882.2934218498742 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:03:01 -# # -------------------------- -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1411.1717435511828 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:04:02 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1411.1717435511828 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:04:08 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1411.1717435511828 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:04:15 -# # ---------------------------- -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -3240.067149411982 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:05:37 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -3240.067149411982 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:06:02 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -3240.067149411982 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:06:13 - -using NeuralPDE, Lux, ModelingToolkit, Optimization, OptimizationOptimJL -import ModelingToolkit: Interval, infimum, supremum - -using NeuralPDE, Flux, OptimizationOptimisers - -function diffeq(u, p, t) - u1, u2 = u - return [u2, p[1] + p[2] * sin(u1) + p[3] * u2] -end -p = [5, -10, -1.7] -u0 = [-1.0, 7.0] -tspan = (0.0, 10.0) -prob = ODEProblem(ODEFunction(diffeq), u0, tspan, p) - -chainnew = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), Flux.Dense(8, 2)) |> - Flux.f64 - -opt = OptimizationOptimisers.Adam(0.1) -opt = Optimisers.ADAGrad(0.1) -opt = Optimisers.AdaMax(0.01) -algnew = NeuralPDE.NNODE(chainnew, opt) -solution_new = solve(prob, algnew, verbose = true, - abstol = 1e-10, maxiters = 7000) -u = reduce(hcat, solution_new.u) -plot(solution_new.t, u[1, :]) -plot!(solution_new.t, u[2, :]) - -algnew = NeuralPDE.BNNODE(chainnew, draw_samples = 200, - n_leapfrog = 30, progress = true) -solution_new = solve(prob, algnew) - -@parameters t -@variables u1(..), u2(..) -D = Differential(t) -eq = [D(u1(t)) ~ u2(t), - D(u2(t)) ~ 5 - 10 * sin(u1(t)) - 1.7 * u2(t)]; - -import ModelingToolkit: Interval -bcs = [u1(0) ~ -1, u2(0) ~ 7] -domains = [t ∈ Interval(0.0, 10.0)] -dt = 0.01 - -input_ = length(domains) # number of dimensions -n = 16 -chain = [Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), Lux.Dense(n, 1)) - for _ in 1:2] - -@named pde_system = PDESystem(eq, bcs, domains, [t], [u1(t), u2(t)]) - -strategy = NeuralPDE.GridTraining(dt) -discretization = PhysicsInformedNN(chain, strategy) -sym_prob = NeuralPDE.symbolic_discretize(pde_system, discretization) - -pde_loss_functions = sym_prob.loss_functions.pde_loss_functions -bc_loss_functions = sym_prob.loss_functions.bc_loss_functions - -callback = function (p, l) - println("loss: ", l) - # println("pde_losses: ", map(l_ -> l_(p), pde_loss_functions)) - # println("bcs_losses: ", map(l_ -> l_(p), bc_loss_functions)) - return false -end - -loss_functions = [pde_loss_functions; bc_loss_functions] - -function loss_function(θ, p) - sum(map(l -> l(θ), loss_functions)) -end - -f_ = OptimizationFunction(loss_function, Optimization.AutoZygote()) -prob = Optimization.OptimizationProblem(f_, sym_prob.flat_init_params) - -res = Optimization.solve(prob, - OptimizationOptimJL.BFGS(); - callback = callback, - maxiters = 1000) -phi = discretization.phi \ No newline at end of file +param1 = sol3lux_pestim.estimated_de_params[1] +@test abs(param1 - p) < abs(0.45 * p) \ No newline at end of file diff --git a/test/BPINN_newform.jl b/test/BPINN_newform.jl new file mode 100644 index 0000000000..fa2f04073e --- /dev/null +++ b/test/BPINN_newform.jl @@ -0,0 +1,4354 @@ +# # Testing Code +using Test, MCMCChains +using ForwardDiff, Distributions, OrdinaryDiffEq +using Flux, OptimizationOptimisers, AdvancedHMC, Lux +using Statistics, Random, Functors, ComponentArrays +using NeuralPDE, MonteCarloMeasurements + +# note that current testing bounds can be easily further tightened but have been inflated for support for Julia build v1 +# on latest Julia version it performs much better for below tests +Random.seed!(100) + +# for sampled params->lux ComponentArray +function vector_to_parameters(ps_new::AbstractVector, ps::NamedTuple) + @assert length(ps_new) == Lux.parameterlength(ps) + i = 1 + function get_ps(x) + z = reshape(view(ps_new, i:(i + length(x) - 1)), size(x)) + i += length(x) + return z + end + return Functors.fmap(get_ps, ps) +end + +## PROBLEM-1 (WITHOUT PARAMETER ESTIMATION) +linear_analytic = (u0, p, t) -> u0 + sin(2 * π * t) / (2 * π) +linear = (u, p, t) -> cos(2 * π * t) +tspan = (0.0, 2.0) +u0 = 0.0 +prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan) +p = prob.p + +# Numerical and Analytical Solutions: testing ahmc_bayesian_pinn_ode() +ta = range(tspan[1], tspan[2], length = 300) +u = [linear_analytic(u0, nothing, ti) for ti in ta] +x̂ = collect(Float64, Array(u) + 0.02 * randn(size(u))) +time = vec(collect(Float64, ta)) +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# testing points for solve() call must match saveat(1/50.0) arg +ta0 = range(tspan[1], tspan[2], length = 101) +u1 = [linear_analytic(u0, nothing, ti) for ti in ta0] +x̂1 = collect(Float64, Array(u1) + 0.02 * randn(size(u1))) +time1 = vec(collect(Float64, ta0)) +physsol0_1 = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] + +chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 +chainlux = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) +init1, re1 = destructure(chainflux) +θinit, st = Lux.setup(Random.default_rng(), chainlux) + +fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainflux, + draw_samples = 2500, + n_leapfrog = 30) + +fh_mcmc_chain2, fhsamples2, fhstats2 = ahmc_bayesian_pinn_ode(prob, chainlux, + draw_samples = 2500, + n_leapfrog = 30) + +# can change training strategies by adding this to call (Quadratuer and GridTraining show good results but stochastics sampling techniques perform bad) +# strategy = QuadratureTraining(; quadrature_alg = QuadGKJL(), +# reltol = 1e-6, +# abstol = 1e-3, maxiters = 1000, +# batch = 0) + +alg = NeuralPDE.BNNODE(chainflux, draw_samples = 2500, + n_leapfrog = 30) +sol1flux = solve(prob, alg) + +alg = NeuralPDE.BNNODE(chainlux, draw_samples = 2500, + n_leapfrog = 30) +sol1lux = solve(prob, alg) + +# testing points +t = time +# Mean of last 500 sampled parameter's curves(flux and lux chains)[Ensemble predictions] +out = re1.(fhsamples1[(end - 500):end]) +yu = collect(out[i](t') for i in eachindex(out)) +fluxmean = [mean(vcat(yu...)[:, i]) for i in eachindex(t)] +meanscurve1 = prob.u0 .+ (t .- prob.tspan[1]) .* fluxmean + +θ = [vector_to_parameters(fhsamples1[i], θinit) for i in 2000:2500] +luxar = [chainlux(t', θ[i], st)[1] for i in 1:500] +luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] +meanscurve2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean + +# --------------------- ahmc_bayesian_pinn_ode() call +@test mean(abs.(x̂ .- meanscurve1)) < 0.05 +@test mean(abs.(physsol1 .- meanscurve1)) < 0.005 +@test mean(abs.(x̂ .- meanscurve2)) < 0.05 +@test mean(abs.(physsol1 .- meanscurve2)) < 0.005 + +#--------------------- solve() call +@test mean(abs.(x̂1 .- sol1flux.ensemblesol[1])) < 0.05 +@test mean(abs.(physsol0_1 .- sol1flux.ensemblesol[1])) < 0.05 +@test mean(abs.(x̂1 .- sol1lux.ensemblesol[1])) < 0.05 +@test mean(abs.(physsol0_1 .- sol1lux.ensemblesol[1])) < 0.05 + +## PROBLEM-1 (WITH PARAMETER ESTIMATION) +linear_analytic = (u0, p, t) -> u0 + sin(p * t) / (p) +linear = (u, p, t) -> cos(p * t) +tspan = (0.0, 2.0) +u0 = 0.0 +p = 2 * pi +prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan, p) + +# Numerical and Analytical Solutions +sol1 = solve(prob, Tsit5(); saveat = 0.01) +u = sol1.u +time = sol1.t + +# BPINN AND TRAINING DATASET CREATION(dataset must be defined only inside problem timespan!) +ta = range(tspan[1], tspan[2], length = 25) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, Array(u) .+ (0.2 .* Array(u) .* randn(size(u)))) +time = vec(collect(Float64, ta)) +dataset = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# testing points for solve call(saveat=1/50.0 ∴ at t = collect(eltype(saveat), prob.tspan[1]:saveat:prob.tspan[2] internally estimates) +ta0 = range(tspan[1], tspan[2], length = 101) +u1 = [linear_analytic(u0, p, ti) for ti in ta0] +x̂1 = collect(Float64, Array(u1) + 0.2 * randn(size(u1))) +time1 = vec(collect(Float64, ta0)) +physsol1_1 = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] + +using Plots, StatsPlots +# plot(dataset[2], calderivatives(dataset)') +yu = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) +plot(yu, [linear_analytic(u0, p, t) for t in yu]) +chainflux1 = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 +chainlux1 = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) +init1, re1 = destructure(chainflux1) +θinit, st = Lux.setup(Random.default_rng(), chainlux1) + +fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainflux1, + dataset = dataset, + draw_samples = 2500, + physdt = 1 / 50.0f0, + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(9, + 0.5), + ], + Metric = DiagEuclideanMetric, + n_leapfrog = 30) + +fh_mcmc_chain2, fhsamples2, fhstats2 = ahmc_bayesian_pinn_ode(prob, chainlux1, + dataset = dataset, + draw_samples = 2500, + physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30) + +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 10.0), + l2std = [0.005], phystd = [0.01], + param = [Normal(11, 6)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30) +# original paper (pure data 0 1) +sol1flux = solve(prob, alg) +sol1flux.estimated_ode_params +# pure data method 1 1 +sol2flux = solve(prob, alg) +sol2flux.estimated_ode_params +# pure data method 1 0 +sol3flux = solve(prob, alg) +sol3flux.estimated_ode_params +# deri collocation +sol4flux = solve(prob, alg) +sol4flux.estimated_ode_params +# collocation +sol5flux = solve(prob, alg) +sol5flux.estimated_ode_params +# collocation + L2Data loss(at 9,0.5 1,2 gives same) +sol6flux = solve(prob, alg) +sol6flux.estimated_ode_params +# 2500 iters +sol7flux = solve(prob, alg) +sol7flux.estimated_ode_params + +plotly() +plot!(yu, sol1flux.ensemblesol[1]) +plot!(yu, sol2flux.ensemblesol[1]) +plot!(yu, sol3flux.ensemblesol[1]) +plot!(yu, sol4flux.ensemblesol[1]) +plot!(yu, sol5flux.ensemblesol[1]) +plot!(yu, sol6flux.ensemblesol[1]) + +plot!(dataset[2], dataset[1]) + +# plot!(sol4flux.ensemblesol[1]) +# plot!(sol5flux.ensemblesol[1]) + +sol2flux.estimated_ode_params + +sol1flux.estimated_ode_params + +sol3flux.estimated_ode_params + +sol4flux.estimated_ode_params + +sol5flux.estimated_ode_params + +alg = NeuralPDE.BNNODE(chainlux1, dataset = dataset, + draw_samples = 2500, + physdt = 1 / 50.0f0, + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(9, + 0.5), + ], + Metric = DiagEuclideanMetric, + n_leapfrog = 30) + +sol2lux = solve(prob, alg) + +# testing points +t = time +# Mean of last 500 sampled parameter's curves(flux and lux chains)[Ensemble predictions] +out = re1.([fhsamples1[i][1:22] for i in 2000:2500]) +yu = collect(out[i](t') for i in eachindex(out)) +fluxmean = [mean(vcat(yu...)[:, i]) for i in eachindex(t)] +meanscurve1 = prob.u0 .+ (t .- prob.tspan[1]) .* fluxmean + +θ = [vector_to_parameters(fhsamples2[i][1:(end - 1)], θinit) for i in 2000:2500] +luxar = [chainlux1(t', θ[i], st)[1] for i in 1:500] +luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] +meanscurve2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean + +# --------------------- ahmc_bayesian_pinn_ode() call +@test mean(abs.(physsol1 .- meanscurve1)) < 0.15 +@test mean(abs.(physsol1 .- meanscurve2)) < 0.15 + +# ESTIMATED ODE PARAMETERS (NN1 AND NN2) +@test abs(p - mean([fhsamples2[i][23] for i in 2000:2500])) < abs(0.25 * p) +@test abs(p - mean([fhsamples1[i][23] for i in 2000:2500])) < abs(0.25 * p) + +#-------------------------- solve() call +@test mean(abs.(physsol1_1 .- sol2flux.ensemblesol[1])) < 8e-2 +@test mean(abs.(physsol1_1 .- sol2lux.ensemblesol[1])) < 8e-2 + +# ESTIMATED ODE PARAMETERS (NN1 AND NN2) +@test abs(p - sol1flux.estimated_ode_params[1]) < abs(0.15 * p) +@test abs(p - sol2lux.estimated_ode_params[1]) < abs(0.15 * p) + +## PROBLEM-2 +linear = (u, p, t) -> u / p + exp(t / p) * cos(t) +tspan = (0.0, 10.0) +u0 = 0.0 +p = -5.0 +prob = ODEProblem(linear, u0, tspan, p) +linear_analytic = (u0, p, t) -> exp(t / p) * (u0 + sin(t)) + +# SOLUTION AND CREATE DATASET +sol = solve(prob, Tsit5(); saveat = 0.1) +u = sol.u +time = sol.t +x̂ = u .+ (u .* 0.2) .* randn(size(u)) +dataset = [x̂, time] +t = sol.t +physsol1 = [linear_analytic(prob.u0, p, t[i]) for i in eachindex(t)] + +ta0 = range(tspan[1], tspan[2], length = 501) +u1 = [linear_analytic(u0, p, ti) for ti in ta0] +time1 = vec(collect(Float64, ta0)) +physsol2 = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] + +chainflux12 = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), + Flux.Dense(6, 1)) |> Flux.f64 +chainlux12 = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), Lux.Dense(6, 1)) +init1, re1 = destructure(chainflux12) +θinit, st = Lux.setup(Random.default_rng(), chainlux12) + +using Flux +using Random + +function derivatives(chainflux, dataset) + loss(x, y) = Flux.mse(chainflux(x), y) + optimizer = Flux.Optimise.ADAM(0.01) + epochs = 2500 + for epoch in 1:epochs + Flux.train!(loss, Flux.params(chainflux), [(dataset[2]', dataset[1]')], optimizer) + end + getgradient(chainflux, dataset) +end + +function getgradient(chainflux, dataset) + return (chainflux(dataset[end]' .+ sqrt(eps(eltype(Float64)))) .- + chainflux(dataset[end]')) ./ + sqrt(eps(eltype(dataset[end][1]))) +end + +ans = derivatives(chainflux12, dataset) + +init3, re = destructure(chainflux12) +init2 == init1 +init3 == init2 +plot!(dataset[end], ans') +plot!(dataset[end], chainflux12(dataset[end]')') + +ars = getgradient(chainflux12, dataset) + +plot!(dataset[end], ars') + +fh_mcmc_chainflux12, fhsamplesflux12, fhstatsflux12 = ahmc_bayesian_pinn_ode(prob, + chainflux12, + draw_samples = 1500, + l2std = [0.03], + phystd = [ + 0.03], + priorsNNw = (0.0, + 10.0), + n_leapfrog = 30) + +fh_mcmc_chainflux22, fhsamplesflux22, fhstatsflux22 = ahmc_bayesian_pinn_ode(prob, + chainflux12, + dataset = dataset, + draw_samples = 1500, + l2std = [0.03], + phystd = [ + 0.03, + ], + priorsNNw = (0.0, + 10.0), + param = [ + Normal(-7, + 4), + ], + n_leapfrog = 30) + +fh_mcmc_chainlux12, fhsampleslux12, fhstatslux12 = ahmc_bayesian_pinn_ode(prob, chainlux12, + draw_samples = 1500, + l2std = [0.03], + phystd = [0.03], + priorsNNw = (0.0, + 10.0), + n_leapfrog = 30) + +fh_mcmc_chainlux22, fhsampleslux22, fhstatslux22 = ahmc_bayesian_pinn_ode(prob, chainlux12, + dataset = dataset, + draw_samples = 1500, + l2std = [0.03], + phystd = [0.03], + priorsNNw = (0.0, + 10.0), + param = [ + Normal(-7, + 4), + ], + n_leapfrog = 30) + +alg1 = NeuralPDE.BNNODE(chainflux12, + dataset = dataset, + draw_samples = 500, + l2std = [0.01], + phystd = [ + 0.03, + ], + priorsNNw = (0.0, + 10.0), + param = [ + Normal(-7, + 4), + ], + n_leapfrog = 30, progress = true) + +# original paper (pure data 0 1) +sol1flux_pestim = solve(prob, alg1) +sol1flux_pestim.estimated_ode_params +# pure data method 1 1 +sol2flux_pestim = solve(prob, alg1) +sol2flux_pestim.estimated_ode_params +# pure data method 1 0 +sol3flux_pestim = solve(prob, alg1) +sol3flux_pestim.estimated_ode_params +# deri collocation +sol4flux_pestim = solve(prob, alg1) +sol4flux_pestim.estimated_ode_params +# collocation +sol5flux_pestim = solve(prob, alg1) +sol5flux_pestim.estimated_ode_params +# collocation + L2Data loss(at 9,0.5 1,2 gives same) +sol6flux_pestim = solve(prob, alg1) +sol6flux_pestim.estimated_ode_params + +using Plots, StatsPlots +ars = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) +plot(time, u) +plot!(ars, sol1flux_pestim.ensemblesol[1]) +plot!(ars, sol2flux_pestim.ensemblesol[1]) +plot!(ars, sol3flux_pestim.ensemblesol[1]) +plot!(ars, sol4flux_pestim.ensemblesol[1]) +plot!(ars, sol5flux_pestim.ensemblesol[1]) +plot!(ars, sol6flux_pestim.ensemblesol[1]) + +sol3flux_pestim.estimated_ode_params + +sol4flux_pestim.estimated_ode_params + +sol5flux_pestim.estimated_ode_params + +sol6flux_pestim.estimated_ode_params + +ars = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) + +init, re1 = destructure(chainflux12) +init +init1 +alg = NeuralPDE.BNNODE(chainlux12, + dataset = dataset, + draw_samples = 1500, + l2std = [0.03], + phystd = [0.03], + priorsNNw = (0.0, + 10.0), + param = [ + Normal(-7, + 4), + ], + n_leapfrog = 30) + +sol3lux_pestim = solve(prob, alg) + +# testing timepoints +t = sol.t +#------------------------------ ahmc_bayesian_pinn_ode() call +# Mean of last 500 sampled parameter's curves(flux chains)[Ensemble predictions] +out = re1.([fhsamplesflux12[i][1:61] for i in 1000:1500]) +yu = [out[i](t') for i in eachindex(out)] +fluxmean = [mean(vcat(yu...)[:, i]) for i in eachindex(t)] +meanscurve1_1 = prob.u0 .+ (t .- prob.tspan[1]) .* fluxmean + +out = re1.([fhsamplesflux22[i][1:61] for i in 1000:1500]) +yu = [out[i](t') for i in eachindex(out)] +fluxmean = [mean(vcat(yu...)[:, i]) for i in eachindex(t)] +meanscurve1_2 = prob.u0 .+ (t .- prob.tspan[1]) .* fluxmean + +@test mean(abs.(sol.u .- meanscurve1_1)) < 1e-2 +@test mean(abs.(physsol1 .- meanscurve1_1)) < 1e-2 +@test mean(abs.(sol.u .- meanscurve1_2)) < 5e-2 +@test mean(abs.(physsol1 .- meanscurve1_2)) < 5e-2 + +# estimated parameters(flux chain) +param1 = mean(i[62] for i in fhsamplesflux22[1000:1500]) +@test abs(param1 - p) < abs(0.3 * p) + +# Mean of last 500 sampled parameter's curves(lux chains)[Ensemble predictions] +θ = [vector_to_parameters(fhsampleslux12[i], θinit) for i in 1000:1500] +luxar = [chainlux12(t', θ[i], st)[1] for i in 1:500] +luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] +meanscurve2_1 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean + +θ = [vector_to_parameters(fhsampleslux22[i][1:(end - 1)], θinit) for i in 1000:1500] +luxar = [chainlux12(t', θ[i], st)[1] for i in 1:500] +luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] +meanscurve2_2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean + +@test mean(abs.(sol.u .- meanscurve2_1)) < 1e-1 +@test mean(abs.(physsol1 .- meanscurve2_1)) < 1e-1 +@test mean(abs.(sol.u .- meanscurve2_2)) < 5e-2 +@test mean(abs.(physsol1 .- meanscurve2_2)) < 5e-2 + +# estimated parameters(lux chain) +param1 = mean(i[62] for i in fhsampleslux22[1000:1500]) +@test abs(param1 - p) < abs(0.3 * p) + +#-------------------------- solve() call +# (flux chain) +@test mean(abs.(physsol2 .- sol3flux_pestim.ensemblesol[1])) < 0.15 +# estimated parameters(flux chain) +param1 = sol3flux_pestim.estimated_ode_params[1] +@test abs(param1 - p) < abs(0.45 * p) + +# (lux chain) +@test mean(abs.(physsol2 .- sol3lux_pestim.ensemblesol[1])) < 0.15 +# estimated parameters(lux chain) +param1 = sol3lux_pestim.estimated_ode_params[1] +@test abs(param1 - p) < abs(0.45 * p) + +using Plots, StatsPlots +using NoiseRobustDifferentiation, Weave, DataInterpolations + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood +# # 25 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, +# draw_samples = 1500, physdt = 1 / 50.0f0, phystd = [0.01], +# l2std = [0.01], +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux1 = solve(prob, alg) +# sol2flux1.estimated_ode_params[1] #6.41722 Particles{Float64, 1}, 6.02404 Particles{Float64, 1} +# sol2flux2 = solve(prob, alg) +# sol2flux2.estimated_ode_params[1] #6.42782 Particles{Float64, 1}, 6.07509 Particles{Float64, 1} +# sol2flux3 = solve(prob, alg) +# sol2flux3.estimated_ode_params[1] #6.42782 Particles{Float64, 1}, 6.00825 Particles{Float64, 1} + +# # 50 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux11 = solve(prob, alg) +# sol2flux11.estimated_ode_params[1] #5.71268 Particles{Float64, 1}, 6.07242 Particles{Float64, 1} +# sol2flux22 = solve(prob, alg) +# sol2flux22.estimated_ode_params[1] #5.74599 Particles{Float64, 1}, 6.04837 Particles{Float64, 1} +# sol2flux33 = solve(prob, alg) +# sol2flux33.estimated_ode_params[1] #5.74599 Particles{Float64, 1}, 6.02838 Particles{Float64, 1} + +# # 100 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux111 = solve(prob, alg) +# sol2flux111.estimated_ode_params[1] #6.59097 Particles{Float64, 1}, 5.89384 Particles{Float64, 1} +# sol2flux222 = solve(prob, alg) +# sol2flux222.estimated_ode_params[1] #6.62813 Particles{Float64, 1}, 5.88216 Particles{Float64, 1} +# sol2flux333 = solve(prob, alg) +# sol2flux333.estimated_ode_params[1] #6.62813 Particles{Float64, 1}, 5.85327 Particles{Float64, 1} + +# # ---------------------------------------------------------- +# # physdt=1/20, full likelihood cdm +# # 25 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux1_cdm = solve(prob, alg) +# sol2flux1_cdm.estimated_ode_params[1]# 6.50506 Particles{Float64, 1} ,6.38963 Particles{Float64, 1} +# sol2flux2_cdm = solve(prob, alg) +# sol2flux2_cdm.estimated_ode_params[1] #6.50032 Particles{Float64, 1} ,6.39817 Particles{Float64, 1} +# sol2flux3_cdm = solve(prob, alg) +# sol2flux3_cdm.estimated_ode_params[1] #6.50032 Particles{Float64, 1} ,6.36296 Particles{Float64, 1} + +# # 50 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux11_cdm = solve(prob, alg) +# sol2flux11_cdm.estimated_ode_params[1] #6.52951 Particles{Float64, 1},5.15621 Particles{Float64, 1} +# sol2flux22_cdm = solve(prob, alg) +# sol2flux22_cdm.estimated_ode_params[1] #6.54988 Particles{Float64, 1},5.16363 Particles{Float64, 1} +# sol2flux33_cdm = solve(prob, alg) +# sol2flux33_cdm.estimated_ode_params[1] #6.54988 Particles{Float64, 1},5.15591 Particles{Float64, 1} + +# # 100 points +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux111_cdm = solve(prob, alg) +# sol2flux111_cdm.estimated_ode_params[1] #6.74338 Particles{Float64, 1}, 9.72422 Particles{Float64, 1} +# sol2flux222_cdm = solve(prob, alg) +# sol2flux222_cdm.estimated_ode_params[1] #6.72642 Particles{Float64, 1}, 9.71991 Particles{Float64, 1} +# sol2flux333_cdm = solve(prob, alg) +# sol2flux333_cdm.estimated_ode_params[1] #6.72642 Particles{Float64, 1}, 9.75045 Particles{Float64, 1} + +# -------------------------------------------------------------------------------------- +# NEW SERIES OF TESTS (IN ORDER OF EXECUTION) +# ------------------------------------------------------------------------------------- +# original paper implementaion +# 25 points +ta = range(tspan[1], tspan[2], length = 25) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, u .+ 0.05 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset1 = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] +# scatter!(time, u) +# dataset +# scatter!(dataset1[2], dataset1[1]) +# plot(time, physsol1) + +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_normal = solve(prob, alg) +sol2flux1_normal.estimated_ode_params[1] #7.70593 Particles{Float64, 1}, 6.36096 Particles{Float64, 1} | 6.45865 Particles{Float64, 1} +sol2flux2_normal = solve(prob, alg) +sol2flux2_normal.estimated_ode_params[1] #6.66347 Particles{Float64, 1}, 6.36974 Particles{Float64, 1} | 6.45865 Particles{Float64, 1} +sol2flux3_normal = solve(prob, alg) +sol2flux3_normal.estimated_ode_params[1] #6.84827 Particles{Float64, 1}, 6.29555 Particles{Float64, 1} | 6.39947 Particles{Float64, 1} + +# 50 points +ta = range(tspan[1], tspan[2], length = 50) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, Array(u) + 0.05 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset2 = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_normal = solve(prob, alg) +sol2flux11_normal.estimated_ode_params[1] #7.83577 Particles{Float64, 1},6.24652 Particles{Float64, 1} | 6.34495 Particles{Float64, 1} +sol2flux22_normal = solve(prob, alg) +sol2flux22_normal.estimated_ode_params[1] #6.49477 Particles{Float64, 1},6.2118 Particles{Float64, 1} | 6.32476 Particles{Float64, 1} +sol2flux33_normal = solve(prob, alg) +sol2flux33_normal.estimated_ode_params[1] #6.47421 Particles{Float64, 1},6.33687 Particles{Float64, 1} | 6.2448 Particles{Float64, 1} + +# 100 points +ta = range(tspan[1], tspan[2], length = 100) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, Array(u) + 0.05 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset3 = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_normal = solve(prob, alg) +sol2flux111_normal.estimated_ode_params[1] #5.96604 Particles{Float64, 1},5.99588 Particles{Float64, 1} | 6.19805 Particles{Float64, 1} +sol2flux222_normal = solve(prob, alg) +sol2flux222_normal.estimated_ode_params[1] #6.05432 Particles{Float64, 1},6.0768 Particles{Float64, 1} | 6.22948 Particles{Float64, 1} +sol2flux333_normal = solve(prob, alg) +sol2flux333_normal.estimated_ode_params[1] #6.08856 Particles{Float64, 1},5.94819 Particles{Float64, 1} | 6.2551 Particles{Float64, 1} + +# LOTKA VOLTERRA CASE +function lotka_volterra(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end + +# initial-value problem. +u01 = [1.0, 1.0] +p1 = [1.5, 1.0, 3.0, 1.0] +tspan1 = (0.0, 6.0) +prob1 = ODEProblem(lotka_volterra, u01, tspan1, p1) + +# chainlux = Lux.Chain(Lux.Dense(1, 7, Lux.tanh), Lux.Dense(7, 7, Lux.tanh), Lux.Dense(7, 2)) +chainflux1 = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), Flux.Dense(8, 2)) + +#testing timepoints must match keyword arg `saveat`` timepoints of solve() call +t1 = collect(Float64, prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]) + +# -------------------------------------------------------------------------- +# original paper implementaion lotka volterra +# 31 points +solution1 = solve(prob1, Tsit5(); saveat = 0.1) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] .+ 0.3 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] .+ 0.3 .* u1[2, :] .* randn(length(u1[2, :])) +dataset2_1 = [x1, y1, time1] +plot(dataset2_1[end], dataset2_1[1]) +plot!(dataset2_1[end], dataset2_1[2]) +plot!(time1, u1[1, :]) +plot!(time1, u1[2, :]) + +alg1 = NeuralPDE.BNNODE(chainflux1, + dataset = dataset2_1, + draw_samples = 1000, + physdt = 1 / 20.0, + l2std = [ + 0.2, + 0.2, + ], + phystd = [ + 0.5, + 0.5, + ], + priorsNNw = (0.0, + 10.0), + param = [ + Normal(4, + 3), + Normal(-2, + 4), + Normal(0, + 5), + Normal(2.5, + 2)], + n_leapfrog = 30, progress = true) + +# original paper (pure data 0 1) +sol1flux1_lotka = solve(prob1, alg1) +sol1flux1_lotka.estimated_ode_params +# pure data method 1 1 +sol2flux1_lotka = solve(prob1, alg1) +sol2flux1_lotka.estimated_ode_params +# pure data method 1 0 +sol3flux1_lotka = solve(prob1, alg1) +sol3flux1_lotka.estimated_ode_params +# deri collocation +sol4flux1_lotka = solve(prob1, alg1) +sol4flux1_lotka.estimated_ode_params +# collocation +sol5flux1_lotka = solve(prob1, alg1) +sol5flux1_lotka.estimated_ode_params +# collocation + L2Data loss(at 9,0.5 1,2 gives same) +sol6flux1_lotka = solve(prob1, alg1) +sol6flux1_lotka.estimated_ode_params + +sol7flux1_lotka = solve(prob1, alg1) +sol7flux1_lotka.estimated_ode_params + +using Plots, StatsPlots +plot(dataset2_1[3], u1[1, :]) +plot!(dataset2_1[3], u1[2, :]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol5flux1_normal.ensemblesol[2]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), + sol1flux1_normal.ensemblesol[1], + legend = :outerbottomleft) +sol1flux2_normal = solve(prob1, alg1) +sol1flux2_normal.estimated_ode_params #| +sol1flux3_normal = solve(prob1, alg1) +sol1flux3_normal.estimated_ode_params #| +sol1flux4_normal = solve(prob1, alg1) +sol1flux4_normal.estimated_ode_params + +plotly() +plot!(title = "yuh") +plot!(dataset2_1[3], dataset2_1[1]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux1_normal.ensemblesol[1]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux2_normal.ensemblesol[1]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux3_normal.ensemblesol[2]) +plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux4_normal.ensemblesol[1]) +plot(time1, u1[1, :]) +plot!(time1, u1[2, :]) + +ars = chainflux1(dataset2_1[end]') +plot(ars[1, :]) +plot!(ars[2, :]) + +function calculate_derivatives(dataset) + u = dataset[1] + u1 = dataset[2] + t = dataset[end] + # control points + n = Int(floor(length(t) / 10)) + # spline for datasetvalues(solution) + # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) + interp = CubicSpline(u, t) + interp1 = CubicSpline(u1, t) + # derrivatives interpolation + dx = t[2] - t[1] + time = collect(t[1]:dx:t[end]) + smoothu = [interp(i) for i in time] + smoothu1 = [interp1(i) for i in time] + # derivative of the spline (must match function derivative) + û = tvdiff(smoothu, 20, 0.5, dx = dx, ε = 1) + û1 = tvdiff(smoothu1, 20, 0.5, dx = dx, ε = 1) + # tvdiff(smoothu, 100, 0.035, dx = dx, ε = 1) + # FDM + # û1 = diff(u) / dx + # dataset[1] and smoothu are almost equal(rounding errors) + return û, û1 + # return 1 +end + +ar = calculate_derivatives(dataset2_1) +plot(ar[1]) +plot!(ar[2]) + +# 61 points +solution1 = solve(prob1, Tsit5(); saveat = 0.1) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +dataset2_2 = [x1, y1, time1] + +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_2, + draw_samples = 1000, + l2std = [ + 0.1, + 0.1, + ], + phystd = [ + 0.1, + 0.1, + ], + priorsNNw = (0.0, + 5.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux11_normal = solve(prob1, alg1) +sol1flux11_normal.estimated_ode_params #| +sol1flux22_normal = solve(prob1, alg1) +sol1flux22_normal.estimated_ode_params #| +sol1flux33_normal = solve(prob1, alg1) +sol1flux33_normal.estimated_ode_params #| + +# 121 points +solution1 = solve(prob1, Tsit5(); saveat = 0.05) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +dataset2_3 = [x1, y1, time1] + +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_3, + draw_samples = 1000, + l2std = [ + 0.1, + 0.1, + ], + phystd = [ + 0.1, + 0.1, + ], + priorsNNw = (0.0, + 5.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux111_normal = solve(prob1, alg1) +sol1flux111_normal.estimated_ode_params #| +sol1flux222_normal = solve(prob1, alg1) +sol1flux222_normal.estimated_ode_params #| +sol1flux333_normal = solve(prob1, alg1) +sol1flux333_normal.estimated_ode_params #| + +# -------------------------------------------------------------------- + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# Sampling 100%|███████████████████████████████| Time: 0:02:30 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# Sampling 100%|███████████████████████████████| Time: 0:01:54 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# Sampling 100%|███████████████████████████████| Time: 0:01:59 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# Sampling 100%|███████████████████████████████| Time: 0:02:44 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# Sampling 100%|███████████████████████████████| Time: 0:02:41 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# Sampling 100%|███████████████████████████████| Time: 0:02:41 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# Sampling 100%|███████████████████████████████| Time: 0:03:52 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# Sampling 100%|███████████████████████████████| Time: 0:03:49 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# Sampling 100%|███████████████████████████████| Time: 0:03:50 + +# # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +# physics Logpdf is : -6.659143464386241e7 +# prior Logpdf is : -150.30074579848434 +# L2lossData Logpdf is : -6.03075717462954e6 +# Sampling 100%|███████████████████████████████| Time: 0:04:54 + +# physics Logpdf is : -8.70012053004202e8 +# prior Logpdf is : -150.3750892952511 +# L2lossData Logpdf is : -6.967914805207133e6 +# Sampling 100%|███████████████████████████████| Time: 0:05:09 + +# physics Logpdf is : -5.417241281343099e7 +# prior Logpdf is : -150.52079555737976 +# L2lossData Logpdf is : -4.195953436792884e6 +# Sampling 100%|███████████████████████████████| Time: 0:05:01 + +# physics Logpdf is : -4.579552981943833e8 +# prior Logpdf is : -150.30491731974283 +# L2lossData Logpdf is : -8.595475827260146e6 +# Sampling 100%|███████████████████████████████| Time: 0:06:08 + +# physics Logpdf is : -1.989281834955769e7 +# prior Logpdf is : -150.16009042727543 +# L2lossData Logpdf is : -1.121270659669029e7 +# Sampling 100%|███████████████████████████████| Time: 0:05:38 + +# physics Logpdf is : -8.683829147264534e8 +# prior Logpdf is : -150.37824872259102 +# L2lossData Logpdf is : -1.0887662888035845e7 +# Sampling 100%|███████████████████████████████| Time: 0:05:50 + +# physics Logpdf is : -3.1944760610332566e8 +# prior Logpdf is : -150.33610348737565 +# L2lossData Logpdf is : -1.215458786744478e7 +# Sampling 100%|███████████████████████████████| Time: 0:10:50 + +# physics Logpdf is : -3.2884572300341567e6 +# prior Logpdf is : -150.21002268156343 +# L2lossData Logpdf is : -1.102536731511176e7 +# Sampling 100%|███████████████████████████████| Time: 0:09:53 + +# physics Logpdf is : -5.31293521002414e8 +# prior Logpdf is : -150.20948536040126 +# L2lossData Logpdf is : -1.818717239584132e7 +# Sampling 100%|███████████████████████████████| Time: 0:08:53 + +# ---------------------------------------------------------- +# Full likelihood no l2 only new L22(NN gradients) +# 25 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_new = solve(prob, alg) +sol2flux1_new.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 6.21662 Particles{Float64, 1} +sol2flux2_new = solve(prob, alg) +sol2flux2_new.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 7.14238 Particles{Float64, 1} +sol2flux3_new = solve(prob, alg) +sol2flux3_new.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 6.79159 Particles{Float64, 1} + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_new = solve(prob, alg) +sol2flux11_new.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 5.33467 Particles{Float64, 1} +sol2flux22_new = solve(prob, alg) +sol2flux22_new.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.52419 Particles{Float64, 1} +sol2flux33_new = solve(prob, alg) +sol2flux33_new.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 5.36921 Particles{Float64, 1} + +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_new = solve(prob, alg) +sol2flux111_new.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.45333 Particles{Float64, 1} +sol2flux222_new = solve(prob, alg) +sol2flux222_new.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 4.64417 Particles{Float64, 1} +sol2flux333_new = solve(prob, alg) +sol2flux333_new.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | 5.88037 Particles{Float64, 1} +# --------------------------------------------------------------------------- + +# ---------------------------------------------------------- +# Full likelihood l2 + new L22(NN gradients) +# 25 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_new_all = solve(prob, alg) +sol2flux1_new_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 6.4358 Particles{Float64, 1} +sol2flux2_new_all = solve(prob, alg) +sol2flux2_new_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 6.52449 Particles{Float64, 1} +sol2flux3_new_all = solve(prob, alg) +sol2flux3_new_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 6.34188 Particles{Float64, 1} + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_new_all = solve(prob, alg) +sol2flux11_new_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 6.37889 Particles{Float64, 1} +sol2flux22_new_all = solve(prob, alg) +sol2flux22_new_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.34747 Particles{Float64, 1} +sol2flux33_new_all = solve(prob, alg) +sol2flux33_new_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 6.39699 Particles{Float64, 1} + +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_new_all = solve(prob, alg) +sol2flux111_new_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.24327 Particles{Float64, 1} +sol2flux222_new_all = solve(prob, alg) +sol2flux222_new_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 6.23928 Particles{Float64, 1} +sol2flux333_new_all = solve(prob, alg) +sol2flux333_new_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | 6.2145 Particles{Float64, 1} + +# --------------------------------------------------------------------------- +# Full likelihood l2 + new L22(dataset gradients) lotka volterra +# 36 points +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_1, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux1_new_all = solve(prob1, alg1) +sol1flux1_new_all.estimated_ode_params[1] #| +sol1flux2_new_all = solve(prob1, alg1) +sol1flux2_new_all.estimated_ode_params[1] #| +sol1flux3_new_all = solve(prob1, alg1) +sol1flux3_new_all.estimated_ode_params[1] #| + +# 61 points +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_2, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux11_new_all = solve(prob1, alg1) +sol1flux11_new_all.estimated_ode_params[1] #| +sol1flux22_new_all = solve(prob1, alg1) +sol1flux22_new_all.estimated_ode_params[1] #| +sol1flux33_new_all = solve(prob1, alg1) +sol1flux33_new_all.estimated_ode_params[1] #| + +# 121 points +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_3, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux111_new_all = solve(prob1, alg1) +sol1flux111_new_all.estimated_ode_params[1] #| +sol1flux222_new_all = solve(prob1, alg1) +sol1flux222_new_all.estimated_ode_params[1] #| +sol1flux333_new_all = solve(prob1, alg1) +sol1flux333_new_all.estimated_ode_params[1] #| +# -------------------------------------------------------------------- + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# L2loss2 Logpdf is : -757.9047847584478 +# Sampling 100%|███████████████████████████████| Time: 0:02:32 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# L2loss2 Logpdf is : -757.9047847584478 +# Sampling 100%|███████████████████████████████| Time: 0:02:19 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -640.4155412187399 +# L2loss2 Logpdf is : -757.9047847584478 +# Sampling 100%|███████████████████████████████| Time: 0:02:31 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# L2loss2 Logpdf is : -1517.3653615845183 +# Sampling 100%|███████████████████████████████| Time: 0:03:45 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# L2loss2 Logpdf is : -1517.3653615845183 +# Sampling 100%|███████████████████████████████| Time: 0:03:20 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1198.9147562830894 +# L2loss2 Logpdf is : -1517.3653615845183 +# Sampling 100%|███████████████████████████████| Time: 0:03:20 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# L2loss2 Logpdf is : -3037.8868319811254 +# Sampling 100%|███████████████████████████████| Time: 0:04:57 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# L2loss2 Logpdf is : -3037.8868319811254 +# Sampling 100%|███████████████████████████████| Time: 0:05:26 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -2473.741390504424 +# L2loss2 Logpdf is : -3037.8868319811254 +# Sampling 100%|███████████████████████████████| Time: 0:05:01 + +# ---------------------------------------------------------- +# Full likelihood l2 + new L22(dataset gradients) +# 25 points +# 1*,2*, +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_newdata_all = solve(prob, alg) +sol2flux1_newdata_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 5.73072 Particles{Float64, 1} +sol2flux2_newdata_all = solve(prob, alg) +sol2flux2_newdata_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 5.71597 Particles{Float64, 1} +sol2flux3_newdata_all = solve(prob, alg) +sol2flux3_newdata_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 5.7313 Particles{Float64, 1} + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_newdata_all = solve(prob, alg) +sol2flux11_newdata_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 6.07153 Particles{Float64, 1} +sol2flux22_newdata_all = solve(prob, alg) +sol2flux22_newdata_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.06623 Particles{Float64, 1} +sol2flux33_newdata_all = solve(prob, alg) +sol2flux33_newdata_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 6.12748 Particles{Float64, 1} + +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_newdata_all = solve(prob, alg) +sol2flux111_newdata_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.26222 Particles{Float64, 1} +sol2flux222_newdata_all = solve(prob, alg) +sol2flux222_newdata_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 5.86494 Particles{Float64, 1} +sol2flux333_newdata_all = solve(prob, alg) +sol2flux333_newdata_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | + +# --------------------------------------------------------------------------- + +# LOTKA VOLTERRA CASE +using Plots, StatsPlots +function lotka_volterra(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end + +# initial-value problem. +u01 = [1.0, 1.0] +p1 = [1.5, 1.0, 3.0, 1.0] +tspan1 = (0.0, 6.0) +prob1 = ODEProblem(lotka_volterra, u01, tspan1, p1) + +chainlux = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), Lux.Dense(6, 2)) + +#testing timepoints must match keyword arg `saveat`` timepoints of solve() call +t1 = collect(Float64, prob.tspan[1]:(1 / 50.0):prob.tspan[2]) + +# -------------------------------------------------------------------------- +# original paper implementaion +# 25 points +solution1 = solve(prob1, Tsit5(); saveat = 0.2) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +dataset2_1 = [x1, y1, time1] + +plot(time1, u1[1, :]) +plot!(time1, u1[2, :]) +scatter!(dataset2_1[3], dataset2_1[1]) +scatter!(dataset2_1[3], dataset2_1[2]) + +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_1, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux1_normal = solve(prob1, alg1) +sol1flux1_normal.estimated_ode_params[1] #| +sol1flux2_normal = solve(prob1, alg1) +sol1flux2_normal.estimated_ode_params[1] #| +sol1flux3_normal = solve(prob1, alg1) +sol1flux3_normal.estimated_ode_params[1] #| + +# 50 points +solution1 = solve(prob1, Tsit5(); saveat = 0.05) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +dataset2_2 = [x1, y1, time1] + +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_2, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux11_normal = solve(prob1, alg1) +sol1flux11_normal.estimated_ode_params[1] #| +sol1flux22_normal = solve(prob1, alg1) +sol1flux22_normal.estimated_ode_params[1] #| +sol1flux33_normal = solve(prob1, alg1) +sol1flux33_normal.estimated_ode_params[1] #| + +# 100 points +solution = solve(prob1, Tsit5(); saveat = 0.05) +time1 = solution1.t +physsol1_1 = solution1.u +u1 = hcat(solution1.u...) +x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) +dataset2_3 = [x1, y1, time1] + +alg1 = NeuralPDE.BNNODE(chainlux, + dataset = dataset2_3, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol1flux111_normal = solve(prob1, alg1) +sol1flux111_normal.estimated_ode_params[1] #| +sol1flux222_normal = solve(prob1, alg1) +sol1flux222_normal.estimated_ode_params[1] #| +sol1flux333_normal = solve(prob1, alg1) +sol1flux333_normal.estimated_ode_params[1] #| + +# -------------------------------------------------------------------- + +# ---------------------------------------------------------- +# Full likelihood no l2 only new L22(NN gradients) +# 25 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_new = solve(prob, alg) +sol2flux1_new.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | +sol2flux2_new = solve(prob, alg) +sol2flux2_new.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | +sol2flux3_new = solve(prob, alg) +sol2flux3_new.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_new = solve(prob, alg) +sol2flux11_new.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | +sol2flux22_new = solve(prob, alg) +sol2flux22_new.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | +sol2flux33_new = solve(prob, alg) +sol2flux33_new.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | + +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_new = solve(prob, alg) +sol2flux111_new.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | +sol2flux222_new = solve(prob, alg) +sol2flux222_new.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | +sol2flux333_new = solve(prob, alg) +sol2flux333_new.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | +# --------------------------------------------------------------------------- + +# ---------------------------------------------------------- +# Full likelihood l2 + new L22(NN gradients) +# 25 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux1_new_all = solve(prob, alg) +sol2flux1_new_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | +sol2flux2_new_all = solve(prob, alg) +sol2flux2_new_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | +sol2flux3_new_all = solve(prob, alg) +sol2flux3_new_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux11_new_all = solve(prob, alg) +sol2flux11_new_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | +sol2flux22_new_all = solve(prob, alg) +sol2flux22_new_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | +sol2flux33_new_all = solve(prob, alg) +sol2flux33_new_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol2flux111_new_all = solve(prob, alg) +sol2flux111_new_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | +sol2flux222_new_all = solve(prob, alg) +sol2flux222_new_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | +sol2flux333_new_all = solve(prob, alg) +sol2flux333_new_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | + +# --------------------------------------------------------------------------- + +# ---------------------------------------------------------- +# Full likelihood l2 + new L22(dataset gradients) +# 25 points +# *1,*2 vs *2.5 +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol1flux1_newdata_all = solve(prob, alg) +sol1flux1_newdata_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | +sol1flux2_newdata_all = solve(prob, alg) +sol1flux2_newdata_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | +sol1flux3_newdata_all = solve(prob, alg) +sol1flux3_newdata_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | + +# 50 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol1flux11_newdata_all = solve(prob, alg) +sol1flux11_newdata_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | +sol1flux22_newdata_all = solve(prob, alg) +sol1flux22_newdata_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | +sol1flux33_newdata_all = solve(prob, alg) +sol1flux33_newdata_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | + +# 100 points +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +sol1flux111_newdata_all = solve(prob, alg) +sol1flux111_newdata_all.estimated_ode_params[1] #| +sol1flux222_newdata_all = solve(prob, alg) +sol1flux222_newdata_all.estimated_ode_params[1] #| +sol1flux333_newdata_all = solve(prob, alg) +sol1flux333_newdata_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | + +# ------------------------------------------------------------------------------------------------------------------------------ + +# sol2flux111.estimated_ode_params[1] +# # mine *5 +# 7.03386Particles{Float64, 1} +# # normal +# 6.38951Particles{Float64, 1} +# 6.67657Particles{Float64, 1} +# # mine *10 +# 7.53672Particles{Float64, 1} +# # mine *2 +# 6.29005Particles{Float64, 1} +# 6.29844Particles{Float64, 1} + +# # new mine *2 +# 6.39008Particles{Float64, 1} +# 6.22071Particles{Float64, 1} +# 6.15611Particles{Float64, 1} + +# # new mine *2 tvdiff(smoothu, 20, 0.035, dx = dx, ε = 1e-2) +# 6.25549Particles{Float64, 1} +# ---------------------------------------------------------- + +# --------------------------------------------------- + +function calculate_derivatives1(dataset) + x̂, time = dataset + num_points = length(x̂) + # Initialize an array to store the derivative values. + derivatives = similar(x̂) + + for i in 2:(num_points - 1) + # Calculate the first-order derivative using central differences. + Δt_forward = time[i + 1] - time[i] + Δt_backward = time[i] - time[i - 1] + + derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) + + derivatives[i] = derivative + end + + # Derivatives at the endpoints can be calculated using forward or backward differences. + derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) + derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) + return derivatives +end + +function calculate_derivatives2(dataset) + u = dataset[1] + t = dataset[2] + # control points + n = Int(floor(length(t) / 10)) + # spline for datasetvalues(solution) + # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) + interp = CubicSpline(u, t) + # derrivatives interpolation + dx = t[2] - t[1] + time = collect(t[1]:dx:t[end]) + smoothu = [interp(i) for i in time] + # derivative of the spline (must match function derivative) + û = tvdiff(smoothu, 20, 0.03, dx = dx, ε = 1) + # tvdiff(smoothu, 100, 0.1, dx = dx) + # + # + # FDM + û1 = diff(u) / dx + # dataset[1] and smoothu are almost equal(rounding errors) + return û, time, smoothu, û1 +end + +# need to do this for all datasets +c = [linear(prob.u0, p, t) for t in dataset3[2]] #ideal case +b = calculate_derivatives1(dataset2) #central diffs +# a = calculate_derivatives2(dataset) #tvdiff(smoothu, 100, 0.1, dx = dx) +d = calculate_derivatives2(dataset1) #tvdiff(smoothu, 20, 0.035, dx = dx, ε = 1e-2) +d = calculate_derivatives2(dataset2) +d = calculate_derivatives2(dataset3) +mean(abs2.(c .- b)) +mean(abs2.(c .- d[1])) +loss(model, x, y) = mean(abs2.(model(x) .- y)); +scatter!(prob.u0 .+ (prob.tspan[2] .- dataset3[2]) .* chainflux1(dataset3[2]')') +loss(chainflux1, dataset3[2]', dataset3[1]') +# mean(abs2.(c[1:24] .- a[4])) +plot(c, label = "ideal deriv") +plot!(b, label = "Centraldiff deriv") +# plot!(a[1], label = "tvdiff(0.1,def) derivatives") +plot!(d[1], label = "tvdiff(0.035,20) derivatives") +plotly() + +# GridTraining , NoiseRobustDiff dataset[2][2]-dataset[2][1] l2std +# 25 points +ta = range(tspan[1], tspan[2], length = 25) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +time1 = collect(tspan[1]:(1 / 50.0):tspan[2]) +physsol = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] +plot(physsol, label = "solution") + +# plots from 32(deriv) +# for d +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 2000, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux1 = solve(prob, alg) +n2_sol2flux1.estimated_ode_params[1] +# with extra likelihood +# 10.2011Particles{Float64, 1} + +# without extra likelihood +# 6.25791Particles{Float64, 1} +# 6.29539Particles{Float64, 1} + +plot!(n2_sol2flux1.ensemblesol[1], label = "tvdiff(0.035,1) derivpar") +plot(dataset[1]) +plot!(physsol1) +# for a +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 2000, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux2 = solve(prob, alg) +n2_sol2flux2.estimated_ode_params[1] +# with extra likelihood +# 8.73602Particles{Float64, 1} +# without extra likelihood + +plot!(n2_sol2flux2.ensemblesol[1], + label = "tvdiff(0.1,def) derivatives", + legend = :outerbottomleft) + +# for b +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 2000, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux3 = solve(prob, alg) +n2_sol2flux3.estimated_ode_params[1] +plot!(n2_sol2flux3.ensemblesol[1], label = "Centraldiff deriv") + +# for c +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 2000, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux4 = solve(prob, alg) +n2_sol2flux4.estimated_ode_params[1] +plot!(n2_sol2flux4.ensemblesol[1], label = "ideal deriv") + +# 50 points + +ta = range(tspan[1], tspan[2], length = 50) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux11 = solve(prob, alg) +n2_sol2flux11.estimated_ode_params[1] + +# 5.90049Particles{Float64, 1} +# 100 points +ta = range(tspan[1], tspan[2], length = 100) +u = [linear_analytic(u0, p, ti) for ti in ta] +x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +time = vec(collect(Float64, ta)) +dataset = [x̂, time] +physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, + draw_samples = 1500, physdt = 1 / 50.0f0, + priorsNNw = (0.0, 3.0), + param = [LogNormal(9, 0.5)], + Metric = DiagEuclideanMetric, + n_leapfrog = 30, progress = true) + +n2_sol2flux111 = solve(prob, alg) +n2_sol2flux111.estimated_ode_params[1] +plot!(n2_sol2flux111.ensemblesol[1]) +8.88555Particles{Float64, 1} + +# 7.15353Particles{Float64, 1} +# 6.21059 Particles{Float64, 1} +# 6.31836Particles{Float64, 1} +0.1 * p +# ---------------------------------------------------------- + +# Gives the linear interpolation value at t=3.5 + +# # Problem 1 with param esimation +# # dataset 0-1 2 percent noise +# p = 6.283185307179586 +# # partial_logdensity +# 6.3549Particles{Float64, 1} +# # full log_density +# 6.34667Particles{Float64, 1} + +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2lux.estimated_ode_params[1] + +# # dataset 0-1 20 percent noise +# # partial log_density +# 6.30244Particles{Float64, 1} +# # full log_density +# 6.24637Particles{Float64, 1} + +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # dataset 0-2 20percent noise +# # partial log_density +# 6.24948Particles{Float64, 1} +# # full log_density +# 6.26095Particles{Float64, 1} + +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# linear_analytic = (u0, p, t) -> u0 + sin(p * t) / (p) +# linear = (u, p, t) -> cos(p * t) +# tspan = (0.0, 2.0) + +# # dataset 0-1 2 percent noise +# p = 6.283185307179586 +# # partial_logdensity +# 6.3549Particles{Float64, 1} +# # full log_density +# 6.34667Particles{Float64, 1} + +# # dataset 0-1 20 percent noise +# # partial log_density +# 6.30244Particles{Float64, 1} +# # full log_density +# 6.24637Particles{Float64, 1} + +# # dataset 0-2 20percent noise +# # partial log_density +# 6.24948Particles{Float64, 1} +# # full log_density +# 6.26095Particles{Float64, 1} + +# # dataset 0-2 20percent noise 50 points(above all are 100 points) +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # partial log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# # i kinda win on 25 points again +# # dataset 0-2 20percent noise 25 points +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # partial log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # i win with 25 points +# # dataset 0-1 20percent noise 25 points +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# # new +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # partial log_density +# sol2flux.estimated_ode_params[1] +# # New +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # (9,2.5)(above are (9,0.5)) +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# # just prev was repeat(just change) +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # partial log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # i lose on 0-1,50 points +# # dataset 0-1 20percent noise 50 points +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] +# # partial log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # (9,2.5) (above are (9,0.5)) +# # FuLL log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # partial log_density +# sol2flux.estimated_ode_params[1] +# sol2flux.estimated_ode_params[1] + +# # ---------------------------------------------------------- +# # Problem 1 with param estimation +# # physdt=1/20, Full likelihood new 0.5*l2std +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n05_sol2flux1 = solve(prob, alg) +# n05_sol2flux1.estimated_ode_params[1] #6.90953 Particles{Float64, 1} +# n05_sol2flux2 = solve(prob, alg) +# n05_sol2flux2.estimated_ode_params[1] #6.82374 Particles{Float64, 1} +# n05_sol2flux3 = solve(prob, alg) +# n05_sol2flux3.estimated_ode_params[1] #6.84465 Particles{Float64, 1} + +# using Plots, StatsPlots +# plot(n05_sol2flux3.ensemblesol[1]) +# plot!(physsol1) +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n05_sol2flux11 = solve(prob, alg) +# n05_sol2flux11.estimated_ode_params[1] #7.0262 Particles{Float64, 1} +# n05_sol2flux22 = solve(prob, alg) +# n05_sol2flux22.estimated_ode_params[1] #5.56438 Particles{Float64, 1} +# n05_sol2flux33 = solve(prob, alg) +# n05_sol2flux33.estimated_ode_params[1] #7.27189 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n05_sol2flux111 = solve(prob, alg) +# n05_sol2flux111.estimated_ode_params[1] #6.90549 Particles{Float64, 1} +# n05_sol2flux222 = solve(prob, alg) +# n05_sol2flux222.estimated_ode_params[1] #5.42436 Particles{Float64, 1} +# n05_sol2flux333 = solve(prob, alg) +# n05_sol2flux333.estimated_ode_params[1] #6.05832 Particles{Float64, 1} + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood new 2*l2std +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2_sol2flux1 = solve(prob, alg) +# n2_sol2flux1.estimated_ode_params[1]#6.9087 Particles{Float64, 1} +# n2_sol2flux2 = solve(prob, alg) +# n2_sol2flux2.estimated_ode_params[1]#6.86507 Particles{Float64, 1} +# n2_sol2flux3 = solve(prob, alg) +# n2_sol2flux3.estimated_ode_params[1]#6.59206 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2_sol2flux11 = solve(prob, alg) +# n2_sol2flux11.estimated_ode_params[1]#7.3715 Particles{Float64, 1} +# n2_sol2flux22 = solve(prob, alg) +# n2_sol2flux22.estimated_ode_params[1]#9.84477 Particles{Float64, 1} +# n2_sol2flux33 = solve(prob, alg) +# n2_sol2flux33.estimated_ode_params[1]#6.87107 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2_sol2flux111 = solve(prob, alg) +# n2_sol2flux111.estimated_ode_params[1]#6.60739 Particles{Float64, 1} +# n2_sol2flux222 = solve(prob, alg) +# n2_sol2flux222.estimated_ode_params[1]#7.05923 Particles{Float64, 1} +# n2_sol2flux333 = solve(prob, alg) +# n2_sol2flux333.estimated_ode_params[1]#6.5017 Particles{Float64, 1} + +# # ---------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood new all 2*l2std +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2all5sol2flux1 = solve(prob, alg) +# n2all5sol2flux1.estimated_ode_params[1]#11.3659 Particles{Float64, 1} +# n2all5sol2flux2 = solve(prob, alg) +# n2all5sol2flux2.estimated_ode_params[1]#6.65634 Particles{Float64, 1} +# n2all5sol2flux3 = solve(prob, alg) +# n2all5sol2flux3.estimated_ode_params[1]#6.61905 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2all5sol2flux11 = solve(prob, alg) +# n2all5sol2flux11.estimated_ode_params[1]#6.27555 Particles{Float64, 1} +# n2all5sol2flux22 = solve(prob, alg) +# n2all5sol2flux22.estimated_ode_params[1]#6.24352 Particles{Float64, 1} +# n2all5sol2flux33 = solve(prob, alg) +# n2all5sol2flux33.estimated_ode_params[1]#6.33723 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n2all5sol2flux111 = solve(prob, alg) +# n2all5sol2flux111.estimated_ode_params[1] #5.95535 Particles{Float64, 1} +# n2all5sol2flux222 = solve(prob, alg) +# n2all5sol2flux222.estimated_ode_params[1] #5.98301 Particles{Float64, 1} +# n2all5sol2flux333 = solve(prob, alg) +# n2all5sol2flux333.estimated_ode_params[1] #5.9081 Particles{Float64, 1} + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood new all (l2+l22) +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nall5sol2flux1 = solve(prob, alg) +# nall5sol2flux1.estimated_ode_params[1]#6.54705 Particles{Float64, 1} +# nall5sol2flux2 = solve(prob, alg) +# nall5sol2flux2.estimated_ode_params[1]#6.6967 Particles{Float64, 1} +# nall5sol2flux3 = solve(prob, alg) +# nall5sol2flux3.estimated_ode_params[1]#6.47173 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nall5sol2flux11 = solve(prob, alg) +# nall5sol2flux11.estimated_ode_params[1]#6.2113 Particles{Float64, 1} +# nall5sol2flux22 = solve(prob, alg) +# nall5sol2flux22.estimated_ode_params[1]#6.10675 Particles{Float64, 1} +# nall5sol2flux33 = solve(prob, alg) +# nall5sol2flux33.estimated_ode_params[1]#6.11541 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nall5sol2flux111 = solve(prob, alg) +# nall5sol2flux111.estimated_ode_params[1]#6.35224 Particles{Float64, 1} +# nall5sol2flux222 = solve(prob, alg) +# nall5sol2flux222.estimated_ode_params[1]#6.40542 Particles{Float64, 1} +# nall5sol2flux333 = solve(prob, alg) +# nall5sol2flux333.estimated_ode_params[1]#6.44206 Particles{Float64, 1} + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood new 5* (new only l22 mod) +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n5sol2flux1 = solve(prob, alg) +# n5sol2flux1.estimated_ode_params[1]#7.05077 Particles{Float64, 1} +# n5sol2flux2 = solve(prob, alg) +# n5sol2flux2.estimated_ode_params[1]#7.07303 Particles{Float64, 1} +# n5sol2flux3 = solve(prob, alg) +# n5sol2flux3.estimated_ode_params[1]#5.10622 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n5sol2flux11 = solve(prob, alg) +# n5sol2flux11.estimated_ode_params[1]#7.39852 Particles{Float64, 1} +# n5sol2flux22 = solve(prob, alg) +# n5sol2flux22.estimated_ode_params[1]#7.30319 Particles{Float64, 1} +# n5sol2flux33 = solve(prob, alg) +# n5sol2flux33.estimated_ode_params[1]#6.73722 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# n5sol2flux111 = solve(prob, alg) +# n5sol2flux111.estimated_ode_params[1]#7.15996 Particles{Float64, 1} +# n5sol2flux222 = solve(prob, alg) +# n5sol2flux222.estimated_ode_params[1]#7.02949 Particles{Float64, 1} +# n5sol2flux333 = solve(prob, alg) +# n5sol2flux333.estimated_ode_params[1]#6.9393 Particles{Float64, 1} + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood new +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nsol2flux1 = solve(prob, alg) +# nsol2flux1.estimated_ode_params[1] #5.82707 Particles{Float64, 1} +# nsol2flux2 = solve(prob, alg) +# nsol2flux2.estimated_ode_params[1] #4.81534 Particles{Float64, 1} +# nsol2flux3 = solve(prob, alg) +# nsol2flux3.estimated_ode_params[1] #5.52965 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nsol2flux11 = solve(prob, alg) +# nsol2flux11.estimated_ode_params[1] #7.04027 Particles{Float64, 1} +# nsol2flux22 = solve(prob, alg) +# nsol2flux22.estimated_ode_params[1] #7.17588 Particles{Float64, 1} +# nsol2flux33 = solve(prob, alg) +# nsol2flux33.estimated_ode_params[1] #6.94495 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# nsol2flux111 = solve(prob, alg) +# nsol2flux111.estimated_ode_params[1] #6.06608 Particles{Float64, 1} +# nsol2flux222 = solve(prob, alg) +# nsol2flux222.estimated_ode_params[1] #6.84726 Particles{Float64, 1} +# nsol2flux333 = solve(prob, alg) +# nsol2flux333.estimated_ode_params[1] #6.83463 Particles{Float64, 1} + +# # ---------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux1 = solve(prob, alg) +# sol2flux1.estimated_ode_params[1] #6.71397 Particles{Float64, 1} 6.37604 Particles{Float64, 1} +# sol2flux2 = solve(prob, alg) +# sol2flux2.estimated_ode_params[1] #6.73509 Particles{Float64, 1} 6.21692 Particles{Float64, 1} +# sol2flux3 = solve(prob, alg) +# sol2flux3.estimated_ode_params[1] #6.65453 Particles{Float64, 1} 6.23153 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux11 = solve(prob, alg) +# sol2flux11.estimated_ode_params[1] #6.23443 Particles{Float64, 1} 6.30635 Particles{Float64, 1} +# sol2flux22 = solve(prob, alg) +# sol2flux22.estimated_ode_params[1] #6.18879 Particles{Float64, 1} 6.30099 Particles{Float64, 1} +# sol2flux33 = solve(prob, alg) +# sol2flux33.estimated_ode_params[1] #6.22773 Particles{Float64, 1} 6.30671 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux111 = solve(prob, alg) +# sol2flux111.estimated_ode_params[1] #6.15832 Particles{Float64, 1} 6.35453 Particles{Float64, 1} +# sol2flux222 = solve(prob, alg) +# sol2flux222.estimated_ode_params[1] #6.16968 Particles{Float64, 1}6.31125 Particles{Float64, 1} +# sol2flux333 = solve(prob, alg) +# sol2flux333.estimated_ode_params[1] #6.12466 Particles{Float64, 1} 6.26514 Particles{Float64, 1} + +# # ---------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, partial likelihood +# # 25 points +# ta = range(tspan[1], tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux1_p = solve(prob, alg) +# sol2flux1_p.estimated_ode_params[1] #5.74065 Particles{Float64, 1} #6.83683 Particles{Float64, 1} +# sol2flux2_p = solve(prob, alg) +# sol2flux2_p.estimated_ode_params[1] #9.82504 Particles{Float64, 1} #6.14568 Particles{Float64, 1} +# sol2flux3_p = solve(prob, alg) +# sol2flux3_p.estimated_ode_params[1] #5.75075 Particles{Float64, 1} #6.08579 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux11_p = solve(prob, alg) +# sol2flux11_p.estimated_ode_params[1] #6.19414 Particles{Float64, 1} #6.04621 Particles{Float64, 1} +# sol2flux22_p = solve(prob, alg) +# sol2flux22_p.estimated_ode_params[1] #6.15227 Particles{Float64, 1} #6.29086 Particles{Float64, 1} +# sol2flux33_p = solve(prob, alg) +# sol2flux33_p.estimated_ode_params[1] #6.19048 Particles{Float64, 1} #6.12516 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol2flux111_p = solve(prob, alg) +# sol2flux111_p.estimated_ode_params[1] #6.51608 Particles{Float64, 1}# 6.42945Particles{Float64, 1} +# sol2flux222_p = solve(prob, alg) +# sol2flux222_p.estimated_ode_params[1] #6.4875 Particles{Float64, 1} # 6.44524Particles{Float64, 1} +# sol2flux333_p = solve(prob, alg) +# sol2flux333_p.estimated_ode_params[1] #6.51679 Particles{Float64, 1}# 6.43152Particles{Float64, 1} + +# # --------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood, dataset(1.0-2.0) +# # 25 points +# ta = range(1.0, tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux1 = solve(prob, alg) +# sol1flux1.estimated_ode_params[1] #6.35164 Particles{Float64, 1} +# sol1flux2 = solve(prob, alg) +# sol1flux2.estimated_ode_params[1] #6.30919 Particles{Float64, 1} +# sol1flux3 = solve(prob, alg) +# sol1flux3.estimated_ode_params[1] #6.33554 Particles{Float64, 1} + +# # 50 points +# ta = range(1.0, tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux11 = solve(prob, alg) +# sol1flux11.estimated_ode_params[1] #6.39769 Particles{Float64, 1} +# sol1flux22 = solve(prob, alg) +# sol1flux22.estimated_ode_params[1] #6.43924 Particles{Float64, 1} +# sol1flux33 = solve(prob, alg) +# sol1flux33.estimated_ode_params[1] #6.4697 Particles{Float64, 1} + +# # 100 points +# ta = range(1.0, tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux111 = solve(prob, alg) +# sol1flux111.estimated_ode_params[1] #6.27812 Particles{Float64, 1} +# sol1flux222 = solve(prob, alg) +# sol1flux222.estimated_ode_params[1] #6.19278 Particles{Float64, 1} +# sol1flux333 = solve(prob, alg) +# sol1flux333.estimated_ode_params[1] # 9.68244Particles{Float64, 1} (first try) # 6.23969 Particles{Float64, 1}(second try) + +# # ---------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, partial likelihood, dataset(1.0-2.0) +# # 25 points +# ta = range(1.0, tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux1_p = solve(prob, alg) +# sol1flux1_p.estimated_ode_params[1]#6.36269 Particles{Float64, 1} + +# sol1flux2_p = solve(prob, alg) +# sol1flux2_p.estimated_ode_params[1]#6.34685 Particles{Float64, 1} + +# sol1flux3_p = solve(prob, alg) +# sol1flux3_p.estimated_ode_params[1]#6.31421 Particles{Float64, 1} + +# # 50 points +# ta = range(1.0, tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux11_p = solve(prob, alg) +# sol1flux11_p.estimated_ode_params[1] #6.15725 Particles{Float64, 1} + +# sol1flux22_p = solve(prob, alg) +# sol1flux22_p.estimated_ode_params[1] #6.18145 Particles{Float64, 1} + +# sol1flux33_p = solve(prob, alg) +# sol1flux33_p.estimated_ode_params[1] #6.21905 Particles{Float64, 1} + +# # 100 points +# ta = range(1.0, tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1flux111_p = solve(prob, alg) +# sol1flux111_p.estimated_ode_params[1]#6.13481 Particles{Float64, 1} + +# sol1flux222_p = solve(prob, alg) +# sol1flux222_p.estimated_ode_params[1]#9.68555 Particles{Float64, 1} + +# sol1flux333_p = solve(prob, alg) +# sol1flux333_p.estimated_ode_params[1]#6.1477 Particles{Float64, 1} + +# # ----------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, partial likelihood, dataset(1-2), again but different density +# # 12 points +# ta = range(1.0, tspan[2], length = 12) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol3flux1_p = solve(prob, alg) +# sol3flux1_p.estimated_ode_params[1]#6.50048 Particles{Float64, 1} +# sol3flux2_p = solve(prob, alg) +# sol3flux2_p.estimated_ode_params[1]#6.57597 Particles{Float64, 1} +# sol3flux3_p = solve(prob, alg) +# sol3flux3_p.estimated_ode_params[1]#6.24487 Particles{Float64, 1} + +# # 25 points +# ta = range(1.0, tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol3flux11_p = solve(prob, alg) +# sol3flux11_p.estimated_ode_params[1]#6.53093 Particles{Float64, 1} + +# sol3flux22_p = solve(prob, alg) +# sol3flux22_p.estimated_ode_params[1]#6.32744 Particles{Float64, 1} + +# sol3flux33_p = solve(prob, alg) +# sol3flux33_p.estimated_ode_params[1]#6.49175 Particles{Float64, 1} + +# # 50 points +# ta = range(1.0, tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol3flux111_p = solve(prob, alg) +# sol3flux111_p.estimated_ode_params[1]#6.4455 Particles{Float64, 1} +# sol3flux222_p = solve(prob, alg) +# sol3flux222_p.estimated_ode_params[1]#6.40736 Particles{Float64, 1} +# sol3flux333_p = solve(prob, alg) +# sol3flux333_p.estimated_ode_params[1]#6.46214 Particles{Float64, 1} + +# # --------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, partial likelihood, dataset(0-1) +# # 25 points +# ta = range(tspan[1], 1.0, length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol0flux1_p = solve(prob, alg) +# sol0flux1_p.estimated_ode_params[1]#7.12625 Particles{Float64, 1} +# sol0flux2_p = solve(prob, alg) +# sol0flux2_p.estimated_ode_params[1]#8.40948 Particles{Float64, 1} +# sol0flux3_p = solve(prob, alg) +# sol0flux3_p.estimated_ode_params[1]#7.18768 Particles{Float64, 1} + +# # 50 points +# ta = range(tspan[1], 1.0, length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol0flux11_p = solve(prob, alg) +# sol0flux11_p.estimated_ode_params[1]#6.23707 Particles{Float64, 1} +# sol0flux22_p = solve(prob, alg) +# sol0flux22_p.estimated_ode_params[1]#6.09728 Particles{Float64, 1} +# sol0flux33_p = solve(prob, alg) +# sol0flux33_p.estimated_ode_params[1]#6.12971 Particles{Float64, 1} + +# # 100 points +# ta = range(tspan[1], 1.0, length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [LogNormal(9, 0.5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol0flux111_p = solve(prob, alg) +# sol0flux111_p.estimated_ode_params[1]#5.99039 Particles{Float64, 1} +# sol0flux222_p = solve(prob, alg) +# sol0flux222_p.estimated_ode_params[1]#5.89609 Particles{Float64, 1} +# sol0flux333_p = solve(prob, alg) +# sol0flux333_p.estimated_ode_params[1]#5.91923 Particles{Float64, 1} + +# # --------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, Full likelihood, dataset(1.0-2.0), Normal(12,5) distri prior +# # 25 points +# ta = range(1.0, tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 6.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f1 = solve(prob, alg) +# sol1f1.estimated_ode_params[1] +# # 10.9818Particles{Float64, 1} +# sol1f2 = solve(prob, alg) +# sol1f2.estimated_ode_params[1] +# # sol1f3 = solve(prob, alg) +# # sol1f3.estimated_ode_params[1] + +# # 50 points +# ta = range(1.0, tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 6.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f11 = solve(prob, alg) +# sol1f11.estimated_ode_params[1] +# sol1f22 = solve(prob, alg) +# sol1f22.estimated_ode_params[1] +# # sol1f33 = solve(prob, alg) +# # sol1f33.estimated_ode_params[1] + +# # 100 points +# ta = range(1.0, tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 6.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f111 = solve(prob, alg) +# sol1f111.estimated_ode_params[1] +# sol1f222 = solve(prob, alg) +# sol1f222.estimated_ode_params[1] +# # sol1f333 = solve(prob, alg) +# # sol1f333.estimated_ode_params[1] + +# # ---------------------------------------------------------- + +# # ---------------------------------------------------------- +# # physdt=1/20, partial likelihood, dataset(1.0-2.0), Normal(12,5) distri prior +# # 25 points +# ta = range(1.0, tspan[2], length = 25) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f1_p = solve(prob, alg) +# sol1f1_p.estimated_ode_params[1] +# sol1f2_p = solve(prob, alg) +# sol1f2_p.estimated_ode_params[1] +# sol1f3_p = solve(prob, alg) +# sol1f3_p.estimated_ode_params[1] + +# # 50 points +# ta = range(1.0, tspan[2], length = 50) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f11_p = solve(prob, alg) +# sol1f11_p.estimated_ode_params[1] +# sol1f22_p = solve(prob, alg) +# sol1f22_p.estimated_ode_params[1] +# sol1f33_p = solve(prob, alg) +# sol1f33_p.estimated_ode_params[1] + +# # 100 points +# ta = range(1.0, tspan[2], length = 100) +# u = [linear_analytic(u0, p, ti) for ti in ta] +# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂, time] +# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] + +# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, +# draw_samples = 1500, physdt = 1 / 50.0f0, +# priorsNNw = (0.0, 3.0), +# param = [Normal(12, 5)], +# Metric = DiagEuclideanMetric, +# n_leapfrog = 30, progress = true) + +# sol1f111_p = solve(prob, alg) +# sol1f111_p.estimated_ode_params[1] +# sol1f222_p = solve(prob, alg) +# sol1f222_p.estimated_ode_params[1] +# sol1f333_p = solve(prob, alg) +# sol1f333_p.estimated_ode_params[1] + +# # ---------------------------------------------------------- + +# plot!(title = "9,2.5 50 training 2>full,1>partial") + +# p +# param1 +# # (lux chain) +# @prob mean(abs.(physsol2 .- sol3lux_pestim.ensemblesol[1])) < 8e-2 + +# # estimated parameters(lux chain) +# param1 = sol3lux_pestim.estimated_ode_params[1] +# @test abs(param1 - p) < abs(0.35 * p) + +# p +# param1 + +# # # my suggested Loss likelihood part +# # # + L2loss2(Tar, θ) +# # # My suggested extra loss function +# # function L2loss2(Tar::LogTargetDensity, θ) +# # f = Tar.prob.f + +# # # parameter estimation chosen or not +# # if Tar.extraparams > 0 +# # dataset = Tar.dataset + +# # # Timepoints to enforce Physics +# # dataset = Array(reduce(hcat, dataset)') +# # t = dataset[end, :] +# # û = dataset[1:(end - 1), :] + +# # ode_params = Tar.extraparams == 1 ? +# # θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : +# # θ[((length(θ) - Tar.extraparams) + 1):length(θ)] + +# # if length(û[:, 1]) == 1 +# # physsol = [f(û[:, i][1], +# # ode_params, +# # t[i]) +# # for i in 1:length(û[1, :])] +# # else +# # physsol = [f(û[:, i], +# # ode_params, +# # t[i]) +# # for i in 1:length(û[1, :])] +# # end +# # #form of NN output matrix output dim x n +# # deri_physsol = reduce(hcat, physsol) + +# # # OG deriv(basically gradient matching in case of an ODEFunction) +# # # in case of PDE or general ODE we would want to reduce residue of f(du,u,p,t) +# # # if length(û[:, 1]) == 1 +# # # deri_sol = [f(û[:, i][1], +# # # Tar.prob.p, +# # # t[i]) +# # # for i in 1:length(û[1, :])] +# # # else +# # # deri_sol = [f(û[:, i], +# # # Tar.prob.p, +# # # t[i]) +# # # for i in 1:length(û[1, :])] +# # # end +# # # deri_sol = reduce(hcat, deri_sol) +# # derivatives = calculate_derivatives(Tar.dataset) +# # deri_sol = reduce(hcat, derivatives) + +# # physlogprob = 0 +# # for i in 1:length(Tar.prob.u0) +# # # can add phystd[i] for u[i] +# # physlogprob += logpdf(MvNormal(deri_physsol[i, :], +# # LinearAlgebra.Diagonal(map(abs2, +# # Tar.l2std[i] .* +# # ones(length(deri_sol[i, :]))))), +# # deri_sol[i, :]) +# # end +# # return physlogprob +# # else +# # return 0 +# # end +# # end + +# # function calculate_derivatives(dataset) +# # x̂, time = dataset +# # num_points = length(x̂) + +# # # Initialize an array to store the derivative values. +# # derivatives = similar(x̂) + +# # for i in 2:(num_points - 1) +# # # Calculate the first-order derivative using central differences. +# # Δt_forward = time[i + 1] - time[i] +# # Δt_backward = time[i] - time[i - 1] + +# # derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) + +# # derivatives[i] = derivative +# # end + +# # # Derivatives at the endpoints can be calculated using forward or backward differences. +# # derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) +# # derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) + +# # return derivatives +# # end + +# size(dataset[1]) +# # Problem 1 with param estimation(flux,lux) +# # Normal +# # 6.20311 Particles{Float64, 1},6.21746Particles{Float64, 1} +# # better +# # 6.29093Particles{Float64, 1}, 6.27925Particles{Float64, 1} +# # Non ideal case +# # 6.14861Particles{Float64, 1}, +# sol2flux.estimated_ode_params +# sol2lux.estimated_ode_params[1] +# p +# size(sol3flux_pestim.ensemblesol[2]) +# plott = sol3flux_pestim.ensemblesol[1] +# using StatsPlots +# plotly() +# plot(t, sol3flux_pestim.ensemblesol[1]) + +# function calculate_derivatives(dataset) +# x̂, time = dataset +# num_points = length(x̂) + +# # Initialize an array to store the derivative values. +# derivatives = similar(x̂) + +# for i in 2:(num_points - 1) +# # Calculate the first-order derivative using central differences. +# Δt_forward = time[i + 1] - time[i] +# Δt_backward = time[i] - time[i - 1] + +# derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) + +# derivatives[i] = derivative +# end + +# # Derivatives at the endpoints can be calculated using forward or backward differences. +# derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) +# derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) + +# return derivatives +# end + +# # Example usage: +# # dataset = [x̂, time] +# derivatives = calculate_derivatives(dataset) +# dataset[1] +# # Access derivative values at specific time points as needed. + +# # # 9,0.5 +# # 0.09894916260292887 +# # 0.09870335436072103 +# # 0.08398556878067913 +# # 0.10109070099105527 +# # 0.09122683737517055 +# # 0.08614958011892977 +# # mean(abs.(x̂ .- meanscurve1)) #0.017112298305523976 +# # mean(abs.(physsol1 .- meanscurve1)) #0.004038636894341354 +# # # 9,4(little worse) +# # mean(abs.(x̂ .- meanscurve1))#0.01800876370000113 +# # mean(abs.(physsol1 .- meanscurve1))#0.007285681280600875 +# # # 30,30 +# # mean(abs.(x̂ .- meanscurve1)) #0.10599926120358046 +# # mean(abs.(physsol1 .- meanscurve1)) #0.10375554193397989 +# # # 30,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.10160824458252521 +# # mean(abs.(physsol1 .- meanscurve1)) #0.09999942538357891 + +# # # ------------------------------------------------normale +# # # 9,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.0333356493928835 +# # mean(abs.(physsol1 .- meanscurve1)) #0.02721733876400459 +# # # 9,4(little worse) +# # mean(abs.(x̂ .- meanscurve1)) #0.020734206709433347 +# # mean(abs.(physsol1 .- meanscurve1)) #0.012502850740700212 +# # # 30,30 +# # mean(abs.(x̂ .- meanscurve1)) #0.10615859683094729 +# # mean(abs.(physsol1 .- meanscurve1)) #0.10508141153722575 +# # # 30,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.10833514946031565 +# # mean(abs.(physsol1 .- meanscurve1)) #0.10668470203219232 + +# # # 9,0.5 +# # 10.158108285475553 +# # 10.207234384538026 +# # 10.215000657664852 +# # 10.213817644016174 +# # 13.380030074088719 +# # 13.348906350967326 + +# # 6.952731422892041 + +# # # All losses +# # 10.161478523326277 +# # # L2 losses 1 +# # 9.33312996960278 +# # # L2 losses 2 +# # 10.217417241370631 + +# # mean([fhsamples1[i][26] for i in 500:1000]) #6.245045767509431 +# # p #6.283185307179586 +# # # 9,4 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.212522300650451 +# # # 30,30 +# # mean([fhsamples1[i][23] for i in 500:1000]) #35.328636809737695 +# # # 30,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #35.232963812125654 + +# # # ---------------------------------------normale +# # # 9,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.547771572198114 +# # p #6.283185307179586 +# # # 9,4 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.158906185002702 +# # # 30,30 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.210400972620185 +# # # 30,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.153845019454522 + +# # # ----------------more dataset normale ----------------------------- +# # # 9,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.271141178216537 +# # p #6.283185307179586 +# # # 9,4 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.241144692919369 +# # # 30,30 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.124480447973127 +# # # 30,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.07838011629903 + +# # # 9,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.016551602015599295 +# # mean(abs.(physsol1 .- meanscurve1)) #0.0021488618484224245 +# # # 9,4(little worse) +# # mean(abs.(x̂ .- meanscurve1)) #0.017022725082640747 +# # mean(abs.(physsol1 .- meanscurve1)) #0.004339761917100232 +# # # 30,30 +# # mean(abs.(x̂ .- meanscurve1)) #0.09668785317864312 +# # mean(abs.(physsol1 .- meanscurve1)) #0.09430712337543362 +# # # 30,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.09958118358974392 +# # mean(abs.(physsol1 .- meanscurve1)) #0.09717454226368502 + +# # # ----------------more dataset special ----------------------------- +# # # 9,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.284355334485365 +# # p #6.283185307179586 +# # # 9,4 +# # mean([fhsamples1[i][23] for i in 500:1000]) #6.259238106698602 +# # # 30,30 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.139808934336987 +# # # 30,0.5 +# # mean([fhsamples1[i][23] for i in 500:1000]) #29.03921327641226 + +# # # 9,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.016627231605546876 +# # mean(abs.(physsol1 .- meanscurve1)) #0.0020311429130039564 +# # # 9,4(little worse) +# # mean(abs.(x̂ .- meanscurve1)) #0.016650324577507352 +# # mean(abs.(physsol1 .- meanscurve1)) #0.0027537543411154677 +# # # 30,30 +# # mean(abs.(x̂ .- meanscurve1)) #0.09713187937270151 +# # mean(abs.(physsol1 .- meanscurve1)) #0.09317278450371556 +# # # 30,0.5 +# # mean(abs.(x̂ .- meanscurve1)) #0.09550234866855814 +# # mean(abs.(physsol1 .- meanscurve1)) #0.09317278450371556 + +# # using Plots, StatsPlots +# # plotly() + +# # --------------------------------------------------------- +# # # # Distribution abstract in wrapper, dataset Float64 +# # # 268.651 s (206393690 allocations: 388.71 GiB) +# # # 318.170551 seconds (206.29 M allocations: 388.453 GiB, 20.83% gc time) + +# # # # Above with dataset Real subtype +# # # 326.201 s (206327409 allocations: 388.42 GiB) +# # # 363.189370 seconds (206.25 M allocations: 387.975 GiB, 15.77% gc time) +# # # 306.171 s (206321277 allocations: 388.55 GiB) +# # # 356.180699 seconds (206.43 M allocations: 388.361 GiB, 13.77% gc time) + +# # # # Above with dataset AbstractFloat subtype +# # # 290.751187 seconds (205.94 M allocations: 387.955 GiB, 12.92% gc time) +# # # 296.319815 seconds (206.38 M allocations: 388.730 GiB, 12.69% gc time) + +# # # # ODEProblem float64 dtaset and vector distri inside +# # # 273.169 s (206128318 allocations: 388.40 GiB) +# # # 274.059531 seconds (205.91 M allocations: 387.953 GiB, 12.77% gc time) + +# # # # Dataset float64 inside and vector distri outsude +# # # 333.603 s (206251143 allocations: 388.41 GiB) +# # # 373.377222 seconds (206.11 M allocations: 387.968 GiB, 13.25% gc time) +# # # 359.745 s (206348301 allocations: 388.41 GiB) +# # # 357.813114 seconds (206.31 M allocations: 388.354 GiB, 13.54% gc time) + +# # # # Dataset float64 inside and vector distri inside +# # # 326.437 s (206253571 allocations: 388.41 GiB) +# # # 290.334083 seconds (205.92 M allocations: 387.954 GiB, 13.82% gc time) + +# # # # current setting +# # # 451.304 s (206476927 allocations: 388.43 GiB) +# # # 384.532732 seconds (206.22 M allocations: 387.976 GiB, 13.17% gc time) +# # # 310.223 s (206332558 allocations: 388.63 GiB) +# # # 344.243889 seconds (206.34 M allocations: 388.409 GiB, 13.84% gc time) +# # # 357.457737 seconds (206.66 M allocations: 389.064 GiB, 18.16% gc time) + +# # # # shit setup +# # # 325.595 s (206283732 allocations: 388.41 GiB) +# # # 334.248753 seconds (206.06 M allocations: 387.964 GiB, 12.60% gc time) +# # # 326.011 s (206370857 allocations: 388.56 GiB) +# # # 327.203339 seconds (206.29 M allocations: 388.405 GiB, 12.92% gc time) + +# # # # in wrapper Distribution prior, insiade FLOAT64 DATASET +# # # 325.158167 seconds (205.97 M allocations: 387.958 GiB, 15.07% gc time) +# # # 429.536 s (206476324 allocations: 388.43 GiB) +# # # 527.364 s (206740343 allocations: 388.58 GiB) + +# # # # wrapper Distribtuion, inside Float64 +# # # 326.017 s (206037971 allocations: 387.96 GiB) +# # # 347.424730 seconds (206.45 M allocations: 388.532 GiB, 12.92% gc time) + +# # # 439.047568 seconds (284.24 M allocations: 392.598 GiB, 15.25% gc time, 14.36% compilation time: 0% of which was recompilation) +# # # 375.472142 seconds (206.40 M allocations: 388.529 GiB, 14.93% gc time) +# # # 374.888820 seconds (206.34 M allocations: 388.346 GiB, 14.09% gc time) +# # # 363.719611 seconds (206.39 M allocations: 388.581 GiB, 15.08% gc time) +# # # # inside Distribtion, instide Float64 +# # # 310.238 s (206324249 allocations: 388.53 GiB) +# # # 308.991494 seconds (206.34 M allocations: 388.549 GiB, 14.01% gc time) +# # # 337.442 s (206280712 allocations: 388.36 GiB) +# # # 299.983096 seconds (206.29 M allocations: 388.512 GiB, 17.14% gc time) + +# # # 394.924357 seconds (206.27 M allocations: 388.337 GiB, 23.68% gc time) +# # # 438.204179 seconds (206.39 M allocations: 388.470 GiB, 23.84% gc time) +# # # 376.626914 seconds (206.46 M allocations: 388.693 GiB, 18.72% gc time) +# # # 286.863795 seconds (206.14 M allocations: 388.370 GiB, 18.80% gc time) +# # # 285.556929 seconds (206.22 M allocations: 388.371 GiB, 17.04% gc time) +# # # 291.471662 seconds (205.96 M allocations: 388.068 GiB, 19.85% gc time) + +# # # 495.814341 seconds (284.62 M allocations: 392.622 GiB, 12.56% gc time, 10.96% compilation time: 0% of which was recompilation) +# # # 361.530617 seconds (206.36 M allocations: 388.526 GiB, 14.98% gc time) +# # # 348.576065 seconds (206.22 M allocations: 388.337 GiB, 15.01% gc time) +# # # 374.575609 seconds (206.45 M allocations: 388.586 GiB, 14.65% gc time) +# # # 314.223008 seconds (206.23 M allocations: 388.411 GiB, 14.63% gc time) + +# # PROBLEM-3 LOTKA VOLTERRA EXAMPLE [WIP] (WITH PARAMETER ESTIMATION)(will be put in tutorial page) +# function lotka_volterra(u, p, t) +# # Model parameters. +# α, β, γ, δ = p +# # Current state. +# x, y = u + +# # Evaluate differential equations. +# dx = (α - β * y) * x # prey +# dy = (δ * x - γ) * y # predator + +# return [dx, dy] +# end + +# u0 = [1.0, 1.0] +# p = [1.5, 1.0, 3.0, 1.0] +# tspan = (0.0, 6.0) +# prob = ODEProblem(lotka_volterra, u0, tspan, p) +# solution = solve(prob, Tsit5(); saveat = 0.05) + +# as = reduce(hcat, solution.u) +# as[1, :] +# # Plot simulation. +# time = solution.t +# u = hcat(solution.u...) +# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct +# x = u[1, :] + 0.5 * randn(length(u[1, :])) +# y = u[2, :] + 0.5 * randn(length(u[1, :])) +# dataset = [x[1:50], y[1:50], time[1:50]] +# # scatter!(time, [x, y]) +# # scatter!(dataset[3], [dataset[2], dataset[1]]) + +# # NN has 2 outputs as u -> [dx,dy] +# chainlux1 = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), +# Lux.Dense(6, 2)) +# chainflux1 = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) + +# # fh_mcmc_chainflux1, fhsamplesflux1, fhstatsflux1 = ahmc_bayesian_pinn_ode(prob, chainflux1, +# # dataset = dataset, +# # draw_samples = 1000, +# # l2std = [ +# # 0.05, +# # 0.05, +# # ], +# # phystd = [ +# # 0.05, +# # 0.05, +# # ], +# # priorsNNw = (0.0, +# # + +# # 3.0)) + +# # check if NN output is more than 1 +# # numoutput = size(luxar[1])[1] +# # if numoutput > 1 +# # # Initialize a vector to store the separated outputs for each output dimension +# # output_matrices = [Vector{Vector{Float32}}() for _ in 1:numoutput] + +# # # Loop through each element in the `as` vector +# # for element in as +# # for i in 1:numoutput +# # push!(output_matrices[i], element[i, :]) # Append the i-th output (i-th row) to the i-th output_matrices +# # end +# # end + +# # ensemblecurves = Vector{}[] +# # for r in 1:numoutput +# # br = hcat(output_matrices[r]...)' +# # ensemblecurve = prob.u0[r] .+ +# # [Particles(br[:, i]) for i in 1:length(t)] .* +# # (t .- prob.tspan[1]) +# # push!(ensemblecurves, ensemblecurve) +# # end + +# # else +# # # ensemblecurve = prob.u0 .+ +# # # [Particles(reduce(vcat, luxar)[:, i]) for i in 1:length(t)] .* +# # # (t .- prob.tspan[1]) +# # print("yuh") +# # end + +# # fhsamplesflux2 +# # nnparams = length(init1) +# # estimnnparams = [Particles(reduce(hcat, fhsamplesflux2)[i, :]) for i in 1:nnparams] +# # ninv=4 +# # estimated_params = [Particles(reduce(hcat, fhsamplesflux2[(end - ninv + 1):end])[i, :]) +# # for i in (nnparams + 1):(nnparams + ninv)] +# # output_matrices[r] +# # br = hcat(output_matrices[r]...)' + +# # br[:, 1] + +# # [Particles(br[:, i]) for i in 1:length(t)] +# # prob.u0 +# # [Particles(br[:, i]) for i in 1:length(t)] .* +# # (t .- prob.tspan[1]) + +# # ensemblecurve = prob.u0[r] .+ +# # [Particles(br[:, i]) for i in 1:length(t)] .* +# # (t .- prob.tspan[1]) +# # push!(ensemblecurves, ensemblecurve) + +# using StatsPlots +# plotly() +# plot(t, ensemblecurve) +# plot(t, ensemblecurves[1]) +# plot!(t, ensemblecurves[2]) +# ensemblecurve +# ensemblecurves[1] +# fh_mcmc_chainflux2, fhsamplesflux2, fhstatsflux2 = ahmc_bayesian_pinn_ode(prob, chainflux1, +# dataset = dataset, +# draw_samples = 1000, +# l2std = [ +# 0.05, +# 0.05, +# ], +# phystd = [ +# 0.05, +# 0.05, +# ], +# priorsNNw = (0.0, +# 3.0), +# param = [ +# Normal(1.5, +# 0.5), +# Normal(1.2, +# 0.5), +# Normal(3.3, +# 0.5), +# Normal(1.4, +# 0.5), +# ], progress = true) + +# alg = NeuralPDE.BNNODE(chainflux1, +# dataset = dataset, +# draw_samples = 1000, +# l2std = [ +# 0.05, +# 0.05, +# ], +# phystd = [ +# 0.05, +# 0.05, +# ], +# priorsNNw = (0.0, +# 3.0), +# param = [ +# Normal(4.5, +# 5), +# Normal(7, +# 2), +# Normal(5, +# 2), +# Normal(-4, +# 6), +# ], +# n_leapfrog = 30, progress = true) + +# sol3flux_pestim = solve(prob, alg) + +# # OG PARAM VALUES +# [1.5, 1.0, 3.0, 1.0] +# # less +# # [1.34, 7.51, 2.54, -2.55] +# # better +# # [1.48, 0.993, 2.77, 0.954] + +# sol3flux_pestim.es +# sol3flux_pestim.estimated_ode_params +# # fh_mcmc_chainlux1, fhsampleslux1, fhstatslux1 = ahmc_bayesian_pinn_ode(prob, chainlux1, +# # dataset = dataset, +# # draw_samples = 1000, +# # l2std = [0.05, 0.05], +# # phystd = [ +# # 0.05, +# # 0.05, +# # ], +# # priorsNNw = (0.0, +# # 3.0)) + +# # fh_mcmc_chainlux2, fhsampleslux2, fhstatslux2 = ahmc_bayesian_pinn_ode(prob, chainlux1, +# # dataset = dataset, +# # draw_samples = 1000, +# # l2std = [0.05, 0.05], +# # phystd = [ +# # 0.05, +# # 0.05, +# # ], +# # priorsNNw = (0.0, +# # 3.0), +# # param = [ +# # Normal(1.5, 0.5), +# # Normal(1.2, 0.5), +# # Normal(3.3, 0.5), +# # Normal(1.4, 0.5), +# # ]) + +# init1, re1 = destructure(chainflux1) +# θinit, st = Lux.setup(Random.default_rng(), chainlux1) +# # PLOT testing points +# t = time +# p = prob.p +# collect(Float64, vcat(ComponentArrays.ComponentArray(θinit))) +# collect(Float64, ComponentArrays.ComponentArray(θinit)) +# # Mean of last 1000 sampled parameter's curves(flux and lux chains)[Ensemble predictions] +# out = re1.([fhsamplesflux1[i][1:68] for i in 500:1000]) +# yu = [out[i](t') for i in eachindex(out)] + +# function getensemble(yu, num_models) +# num_rows, num_cols = size(yu[1]) +# row_means = zeros(Float32, num_rows, num_cols) +# for i in 1:num_models +# row_means .+= yu[i] +# end +# row_means ./ num_models +# end +# fluxmean = getensemble(yu, length(out)) +# meanscurve1_1 = prob.u0 .+ (t' .- prob.tspan[1]) .* fluxmean +# mean(abs.(u .- meanscurve1_1)) + +# plot!(t, physsol1) +# @test mean(abs2.(x̂ .- meanscurve1_1)) < 2e-2 +# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 +# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 +# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 + +# out = re1.([fhsamplesflux2[i][1:68] for i in 500:1000]) +# yu = collect(out[i](t') for i in eachindex(out)) +# fluxmean = getensemble(yu, length(out)) +# meanscurve1_2 = prob.u0 .+ (t' .- prob.tspan[1]) .* fluxmean +# mean(abs.(u .- meanscurve1_2)) + +# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 +# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 +# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 +# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 + +# θ = [vector_to_parameters(fhsampleslux1[i][1:(end - 4)], θinit) for i in 500:1000] +# luxar = [chainlux1(t', θ[i], st)[1] for i in 1:500] +# luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] +# meanscurve2_1 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean + +# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 +# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 +# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 +# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 + +# θ = [vector_to_parameters(fhsampleslux2[i][1:(end - 4)], θinit) for i in 500:1000] +# luxar = [chainlux1(t', θ[i], st)[1] for i in 1:500] +# luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] +# meanscurve2_2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean + +# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 +# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 +# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 +# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 + +# # # ESTIMATED ODE PARAMETERS (NN1 AND NN2) +# @test abs(p - mean([fhsamplesflux2[i][69] for i in 500:1000])) < 0.1 * p[1] +# @test abs(p - mean([fhsampleslux2[i][69] for i in 500:1000])) < 0.2 * p[1] + +# # @test abs(p - mean([fhsamplesflux2[i][70] for i in 500:1000])) < 0.1 * p[2] +# # @test abs(p - mean([fhsampleslux2[i][70] for i in 500:1000])) < 0.2 * p[2] + +# # @test abs(p - mean([fhsamplesflux2[i][71] for i in 500:1000])) < 0.1 * p[3] +# # @test abs(p - mean([fhsampleslux2[i][71] for i in 500:1000])) < 0.2 * p[3] + +# # @test abs(p - mean([fhsamplesflux2[i][72] for i in 500:1000])) < 0.1 * p[4] +# # @test abs(p - mean([fhsampleslux2[i][72] for i in 500:1000])) < 0.2 * p[4] + +# # fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainlux1, +# # dataset = dataset, +# # draw_samples = 1000, +# # l2std = [0.05, 0.05], +# # phystd = [0.05, 0.05], +# # priorsNNw = (0.0, 3.0), +# # param = [ +# # Normal(1.5, 0.5), +# # Normal(1.2, 0.5), +# # Normal(3.3, 0.5), +# # Normal(1.4, 0.5), +# # ], autodiff = true) + +# # fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainlux1, +# # dataset = dataset, +# # draw_samples = 1000, +# # l2std = [0.05, 0.05], +# # phystd = [0.05, 0.05], +# # priorsNNw = (0.0, 3.0), +# # param = [ +# # Normal(1.5, 0.5), +# # Normal(1.2, 0.5), +# # Normal(3.3, 0.5), +# # Normal(1.4, 0.5), +# # ], nchains = 2) + +# # NOTES (WILL CLEAR LATER) +# # -------------------------------------------------------------------------------------------- +# # Hamiltonian energy must be lowest(more paramters the better is it to map onto them) +# # full better than L2 and phy individual(test) +# # in mergephys more points after training points is better from 20->40 +# # does consecutive runs bceome better? why?(plot 172)(same chain maybe) +# # does density of points in timespan matter dataset vs internal timespan?(plot 172)(100+0.01) +# # when training from 0-1 and phys from 1-5 with 1/150 simple nn slow,but bigger nn faster decrease in Hmailtonian +# # bigger time interval more curves to adapt to only more parameters adapt to that, better NN architecture +# # higher order logproblems solve better +# # repl up up are same instances? but reexecute calls are new? + +# #Compare results against paper example +# # Lux chains support (DONE) +# # fix predictions for odes depending upon 1,p in f(u,p,t)(DONE) +# # lotka volterra learn curve beyond l2 losses(L2 losses determine accuracy of parameters)(parameters cant run free ∴ L2 interval only) +# # check if prameters estimation works(YES) +# # lotka volterra parameters estimate (DONE) + +# using NeuralPDE, Lux, Flux, Optimization, OptimizationOptimJL +# import ModelingToolkit: Interval +# using Plots, StatsPlots +# plotly() +# # Profile.init() + +# @parameters x y +# @variables u(..) +# Dxx = Differential(x)^2 +# Dyy = Differential(y)^2 + +# # 2D PDE +# eq = Dxx(u(x, y)) + Dyy(u(x, y)) ~ -sin(pi * x) * sin(pi * y) + +# # Boundary conditions +# bcs = [u(0, y) ~ 0.0, u(1, y) ~ 0.0, +# u(x, 0) ~ 0.0, u(x, 1) ~ 0.0] +# # Space and time domains +# domains = [x ∈ Interval(0.0, 1.0), +# y ∈ Interval(0.0, 1.0)] + +# # Neural network +# dim = 2 # number of dimensions +# chain = Flux.Chain(Flux.Dense(dim, 16, Lux.σ), Flux.Dense(16, 16, Lux.σ), Flux.Dense(16, 1)) +# θ, re = destructure(chain) +# # Discretization +# dx = 0.05 +# discretization = PhysicsInformedNN(chain, GridTraining(dx)) + +# @named pde_system = PDESystem(eq, bcs, domains, [x, y], [u(x, y)]) + +# pinnrep = symbolic_discretize(pde_system, discretization) +# typeof(pinnrep.phi) +# typeof(pinnrep.phi) +# typeof(re) +# pinnrep.phi([1, 2], θ) + +# typeof(θ) + +# print(pinnrep) +# pinnrep.eqs +# pinnrep.bcs +# pinnrep.domains +# pinnrep.eq_params +# pinnrep.defaults +# print(pinnrep.default_p) +# pinnrep.param_estim +# print(pinnrep.additional_loss) +# pinnrep.adaloss +# pinnrep.depvars +# pinnrep.indvars +# pinnrep.dict_depvar_input +# pinnrep.dict_depvars +# pinnrep.dict_indvars +# print(pinnrep.logger) +# pinnrep.multioutput +# pinnrep.iteration +# pinnrep.init_params +# pinnrep.flat_init_params +# pinnrep.phi +# pinnrep.derivative +# pinnrep.strategy +# pinnrep.pde_indvars +# pinnrep.bc_indvars +# pinnrep.pde_integration_vars +# pinnrep.bc_integration_vars +# pinnrep.integral +# pinnrep.symbolic_pde_loss_functions +# pinnrep.symbolic_bc_loss_functions +# pinnrep.loss_functions + +# # = discretize(pde_system, discretization) +# prob = symbolic_discretize(pde_system, discretization) +# # "The boundary condition loss functions" +# sum([prob.loss_functions.bc_loss_functions[i](θ) for i in eachindex(1:4)]) +# sum([prob.loss_functions.pde_loss_functions[i](θ) for i in eachindex(1)]) + +# prob.loss_functions.full_loss_function(θ, 32) + +# prob.loss_functions.bc_loss_functions[1](θ) + +# prob.loss_functions.bc_loss_functions +# prob.loss_functions.full_loss_function +# prob.loss_functions.additional_loss_function +# prob.loss_functions.pde_loss_functions + +# 1.3953060473003345 + 1.378102161087438 + 1.395376727128639 + 1.3783868705075002 + +# 0.22674532775196876 +# # "The PDE loss functions" +# prob.loss_functions.pde_loss_functions +# prob.loss_functions.pde_loss_functions[1](θ) +# # "The full loss function, combining the PDE and boundary condition loss functions.This is the loss function that is used by the optimizer." +# prob.loss_functions.full_loss_function(θ, nothing) +# prob.loss_functions.full_loss_function(θ, 423423) + +# # "The wrapped `additional_loss`, as pieced together for the optimizer." +# prob.loss_functions.additional_loss_function +# # "The pre-data version of the PDE loss function" +# prob.loss_functions.datafree_pde_loss_functions +# # "The pre-data version of the BC loss function" +# prob.loss_functions.datafree_bc_loss_functions + +# using Random +# θ, st = Lux.setup(Random.default_rng(), chain) +# #Optimizer +# opt = OptimizationOptimJL.BFGS() + +# #Callback function +# callback = function (p, l) +# println("Current loss is: $l") +# return false +# end + +# res = Optimization.solve(prob, opt, callback = callback, maxiters = 1000) +# phi = discretization.phi + +# # ------------------------------------------------ +# using NeuralPDE, Lux, ModelingToolkit, Optimization, OptimizationOptimJL, OrdinaryDiffEq, +# Plots +# import ModelingToolkit: Interval, infimum, supremum +# @parameters t, σ_, β, ρ +# @variables x(..), y(..), z(..) +# Dt = Differential(t) +# eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), +# Dt(y(t)) ~ x(t) * (ρ - z(t)) - y(t), +# Dt(z(t)) ~ x(t) * y(t) - β * z(t)] + +# bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] +# domains = [t ∈ Interval(0.0, 1.0)] +# dt = 0.01 + +# input_ = length(domains) +# n = 8 +# chain1 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, 1)) +# chain2 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, 1)) +# chain3 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, n, Lux.σ), +# Lux.Dense(n, 1)) + +# function lorenz!(du, u, p, t) +# du[1] = 10.0 * (u[2] - u[1]) +# du[2] = u[1] * (28.0 - u[3]) - u[2] +# du[3] = u[1] * u[2] - (8 / 3) * u[3] +# end + +# u0 = [1.0; 0.0; 0.0] +# tspan = (0.0, 1.0) +# prob = ODEProblem(lorenz!, u0, tspan) +# sol = solve(prob, Tsit5(), dt = 0.1) +# ts = [infimum(d.domain):dt:supremum(d.domain) for d in domains][1] +# function getData(sol) +# data = [] +# us = hcat(sol(ts).u...) +# ts_ = hcat(sol(ts).t...) +# return [us, ts_] +# end +# data = getData(sol) + +# (u_, t_) = data +# len = length(data[2]) + +# depvars = [:x, :y, :z] +# function additional_loss(phi, θ, p) +# return sum(sum(abs2, phi[i](t_, θ[depvars[i]]) .- u_[[i], :]) / len for i in 1:1:3) +# end + +# discretization = NeuralPDE.PhysicsInformedNN([chain1, chain2, chain3], +# NeuralPDE.GridTraining(dt), +# param_estim = false, +# additional_loss = additional_loss) +# @named pde_system = PDESystem(eqs, bcs, domains, [t], [x(t), y(t), z(t)], [σ_, ρ, β], +# defaults = Dict([p .=> 1.0 for p in [σ_, ρ, β]])) +# prob = NeuralPDE.discretize(pde_system, discretization) +# callback = function (p, l) +# println("Current loss is: $l") +# return false +# end +# res = Optimization.solve(prob, BFGS(); callback = callback, maxiters = 5000) +# p_ = res.u[(end - 2):end] # p_ = [9.93, 28.002, 2.667] + +# minimizers = [res.u.depvar[depvars[i]] for i in 1:3] +# ts = [infimum(d.domain):(dt / 10):supremum(d.domain) for d in domains][1] +# u_predict = [[discretization.phi[i]([t], minimizers[i])[1] for t in ts] for i in 1:3] +# plot(sol) +# plot!(ts, u_predict, label = ["x(t)" "y(t)" "z(t)"]) + +# discretization.multioutput +# discretization.chain +# discretization.strategy +# discretization.init_params +# discretization.phi +# discretization.derivative +# discretization.param_estim +# discretization.additional_loss +# discretization.adaptive_loss +# discretization.logger +# discretization.log_options +# discretization.iteration +# discretization.self_increment +# discretization.multioutput +# discretization.kwargs + +# struct BNNODE1{P <: Vector{<:Distribution}} +# chain::Any +# Kernel::Any +# draw_samples::UInt32 +# priorsNNw::Tuple{Float64, Float64} +# param::P +# l2std::Vector{Float64} +# phystd::Vector{Float64} + +# function BNNODE1(chain, Kernel; draw_samples = 2000, priorsNNw = (0.0, 3.0), param = [], +# l2std = [0.05], phystd = [0.05]) +# BNNODE1(chain, Kernel, draw_samples, priorsNNw, param, l2std, phystd) +# end +# end + +# struct BNNODE3{C, K, P <: Union{Any, Vector{<:Distribution}}} +# chain::C +# Kernel::K +# draw_samples::UInt32 +# priorsNNw::Tuple{Float64, Float64} +# param::P +# l2std::Vector{Float64} +# phystd::Vector{Float64} + +# function BNNODE3(chain, Kernel; draw_samples = 2000, priorsNNw = (0.0, 3.0), param = [], +# l2std = [0.05], phystd = [0.05]) +# new{typeof(chain), typeof(Kernel), typeof(param)}(chain, Kernel, draw_samples, +# priorsNNw, param, l2std, phystd) +# end +# end +# linear_analytic = (u0, p, t) -> u0 + sin(2 * π * t) / (2 * π) +# linear = (u, p, t) -> cos(2 * π * t) +# tspan = (0.0, 2.0) +# u0 = 0.0 +# prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan) + +# ta = range(tspan[1], tspan[2], length = 300) +# u = [linear_analytic(u0, nothing, ti) for ti in ta] +# sol1 = solve(prob, Tsit5()) + +# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct +# x̂ = collect(Float64, Array(u) + 0.02 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂[1:100], time[1:100]] + +# # Call BPINN, create chain +# chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) +# chainlux = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) +# HMC +# solve(prob, BNNODE(chainflux, HMC)) +# BNNODE1(chainflux, HMC, 2000) + +# draw_samples = 2000 +# priorsNNw = (0.0, 3.0) +# param = [] +# l2std = [0.05] +# phystd = [0.05] +# @time BNNODE3(chainflux, HMC, draw_samples = 2000, priorsNNw = (0.0, 3.0), +# param = [nothing], +# l2std = [0.05], phystd = [0.05]) +# typeof(Nothing) <: Vector{<:Distribution} +# Nothing <: Distribution +# {UnionAll} <: Distribution +# @time [Nothing] +# typeof([Nothing]) +# @time [1] + +# function test1(sum; c = 23, d = 32) +# return sum + c + d +# end +# function test(a, b; c, d) +# return test1(a + b, c, d) +# end + +# test(2, 2) + +# struct BNNODE3{C, K, P <: Union{Vector{Nothing}, Vector{<:Distribution}}} +# chain::C +# Kernel::K +# draw_samples::Int64 +# priorsNNw::Tuple{Float64, Float64} +# param::P +# l2std::Vector{Float64} +# phystd::Vector{Float64} + +# function BNNODE3(chain, Kernel; draw_samples, +# priorsNNw, param = [nothing], l2std, phystd) +# new{typeof(chain), typeof(Kernel), typeof(param)}(chain, +# Kernel, +# draw_samples, +# priorsNNw, +# param, l2std, +# phystd) +# end +# end + +# function solve1(prob::DiffEqBase.AbstractODEProblem, alg::BNNODE3; +# dataset = [nothing], dt = 1 / 20.0, +# init_params = nothing, nchains = 1, +# autodiff = false, Integrator = Leapfrog, +# Adaptor = StanHMCAdaptor, targetacceptancerate = 0.8, +# Metric = DiagEuclideanMetric, jitter_rate = 3.0, +# tempering_rate = 3.0, max_depth = 10, Δ_max = 1000, +# n_leapfrog = 10, δ = 0.65, λ = 0.3, progress = true, +# verbose = false) +# chain = alg.chain +# l2std = alg.l2std +# phystd = alg.phystd +# priorsNNw = alg.priorsNNw +# Kernel = alg.Kernel +# draw_samples = alg.draw_samples + +# param = alg.param == [nothing] ? [] : alg.param +# mcmcchain, samples, statistics = ahmc_bayesian_pinn_ode(prob, chain, dataset = dataset, +# draw_samples = draw_samples, +# init_params = init_params, +# physdt = dt, l2std = l2std, +# phystd = phystd, +# priorsNNw = priorsNNw, +# param = param, +# nchains = nchains, +# autodiff = autodiff, +# Kernel = Kernel, +# Integrator = Integrator, +# Adaptor = Adaptor, +# targetacceptancerate = targetacceptancerate, +# Metric = Metric, +# jitter_rate = jitter_rate, +# tempering_rate = tempering_rate, +# max_depth = max_depth, +# Δ_max = Δ_max, +# n_leapfrog = n_leapfrog, δ = δ, +# λ = λ, progress = progress, +# verbose = verbose) +# end + +# linear_analytic = (u0, p, t) -> u0 + sin(2 * π * t) / (2 * π) +# linear = (u, p, t) -> cos(2 * π * t) +# tspan = (0.0, 2.0) +# u0 = 0.0 +# prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan) + +# ta = range(tspan[1], tspan[2], length = 300) +# u = [linear_analytic(u0, nothing, ti) for ti in ta] +# # sol1 = solve(prob, Tsit5()) + +# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct +# x̂ = collect(Float64, Array(u) + 0.02 * randn(size(u))) +# time = vec(collect(Float64, ta)) +# dataset = [x̂[1:100], time[1:100]] + +# # Call BPINN, create chain +# chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) +# chainlux = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) +# HMC + +# solve1(prob, a) +# a = BNNODE3(chainflux, HMC, draw_samples = 2000, +# priorsNNw = (0.0, 3.0), +# l2std = [0.05], phystd = [0.05]) + +# Define Lotka-Volterra model. +function lotka_volterra1(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end + +u0 = [1.0, 1.0] +p = [1.5, 1.0, 3.0, 1.0] +tspan = (0.0, 6.0) +prob = ODEProblem(lotka_volterra1, u0, tspan, p) +solution = solve(prob, Tsit5(); saveat = 0.05) + +as = reduce(hcat, solution.u) +as[1, :] +# Plot simulation. +time = solution.t +u = hcat(solution.u...) +# BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct +x = u[1, :] + 0.5 * randn(length(u[1, :])) +y = u[2, :] + 0.5 * randn(length(u[1, :])) +dataset = [x[1:50], y[1:50], time[1:50]] +# scatter!(time, [x, y]) +# scatter!(dataset[3], [dataset[2], dataset[1]]) + +# NN has 2 outputs as u -> [dx,dy] +chainlux1 = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), + Lux.Dense(6, 2)) +chainflux1 = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) + +fh_mcmc_chainflux1, fhsamplesflux1, fhstatsflux1 = ahmc_bayesian_pinn_ode(prob, chainflux1, + dataset = dataset, + draw_samples = 1000, + l2std = [ + 0.05, + 0.05, + ], + phystd = [ + 0.05, + 0.05, + ], + priorsNNw = (0.0, 3.0), progress = true) +ahmc_bayesian_pinn_ode(prob, chainflux1, + dataset = dataset, + draw_samples = 1000, + l2std = [ + 0.05, + 0.05, + ], + phystd = [ + 0.05, + 0.05, + ], + priorsNNw = (0.0, 3.0), progress = true) + +# 2×171 Matrix{Float64}: +# -0.5 -0.518956 -0.529639 … -1.00266 -1.01049 +# 2.0 1.97109 1.92747 0.42619 0.396335 + +# 2-element Vector{Float64}: +# -119451.94949911036 +# -128543.23714618056 + +# alg = NeuralPDE.BNNODE(chainflux1, +# dataset = dataset, +# draw_samples = 1000, +# l2std = [ +# 0.05, +# 0.05, +# ], +# phystd = [ +# 0.05, +# 0.05, +# ], +# priorsNNw = (0.0, +# 3.0), +# param = [ +# Normal(4.5, +# 5), +# Normal(7, +# 2), +# Normal(5, +# 2), +# Normal(-4, +# 6), +# ], +# n_leapfrog = 30, progress = true) + +# sol3flux_pestim = solve(prob, alg) + +# ---------------------------------------------- +# original paper implementation +# 25 points +run1 #7.70593 Particles{Float64, 1} +run2 #6.66347 Particles{Float64, 1} +run3 #6.84827 Particles{Float64, 1} + +# 50 points +run1 #7.83577 Particles{Float64, 1} +run2 #6.49477 Particles{Float64, 1} +run3 #6.47421 Particles{Float64, 1} + +# 100 points +run1 #5.96604 Particles{Float64, 1} +run2 #6.05432 Particles{Float64, 1} +run3 #6.08856 Particles{Float64, 1} + +# Full likelihood(uses total variation regularized differentiation) +# 25 points +run1 #6.41722 Particles{Float64, 1} +run2 #6.42782 Particles{Float64, 1} +run3 #6.42782 Particles{Float64, 1} + +# 50 points +run1 #5.71268 Particles{Float64, 1} +run2 #5.74599 Particles{Float64, 1} +run3 #5.74599 Particles{Float64, 1} + +# 100 points +run1 #6.59097 Particles{Float64, 1} +run2 #6.62813 Particles{Float64, 1} +run3 #6.62813 Particles{Float64, 1} + +using Plots, StatsPlots +function lotka_volterra(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end + +# initial-value problem. +u0 = [1.0, 1.0] +p = [1.5, 1.0, 3.0, 1.0] +tspan = (0.0, 6.0) +prob = ODEProblem(lotka_volterra, u0, tspan, p) + +# Plot simulation. + +solution = solve(prob, Tsit5(); saveat = 0.05) +plot(solve(prob, Tsit5())) + +# Dataset creation for parameter estimation +time = solution.t +u = hcat(solution.u...) +x = u[1, :] + 0.5 * randn(length(u[1, :])) +y = u[2, :] + 0.5 * randn(length(u[1, :])) +dataset = [x, y, time] + +# Neural Networks must have 2 outputs as u -> [dx,dy] in function lotka_volterra() +chainflux = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) |> + Flux.f64 + +chainlux = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), Lux.Dense(6, 2)) + +alg1 = NeuralPDE.BNNODE(chainflux, + dataset = dataset, + draw_samples = 1000, + l2std = [ + 0.01, + 0.01, + ], + phystd = [ + 0.01, + 0.01, + ], + priorsNNw = (0.0, + 3.0), + param = [ + LogNormal(1.5, + 0.5), + LogNormal(1.2, + 0.5), + LogNormal(3.3, + 1), + LogNormal(1.4, + 1)], + n_leapfrog = 30, progress = true) + +sol_flux_pestim = solve(prob, alg1) + +# Dataset not needed as we are solving the equation with ideal parameters +alg2 = NeuralPDE.BNNODE(chainlux, + draw_samples = 1000, + l2std = [ + 0.05, + 0.05, + ], + phystd = [ + 0.05, + 0.05, + ], + priorsNNw = (0.0, + 3.0), + n_leapfrog = 30, progress = true) + +sol_lux = solve(prob, alg2) + +#testing timepoints must match keyword arg `saveat`` timepoints of solve() call +t = collect(Float64, prob.tspan[1]:(1 / 50.0):prob.tspan[2]) + +# plotting solution for x,y for chain_flux +plot(t, sol_flux_pestim.ensemblesol[1]) +plot!(t, sol_flux_pestim.ensemblesol[2]) + +plot(sol_flux_pestim.ens1mblesol[1]) +plot!(sol_flux_pestim.ensemblesol[2]) + +# estimated ODE parameters by .estimated_ode_params, weights and biases by .estimated_nn_params +sol_flux_pestim.estimated_nn_params +sol_flux_pestim.estimated_ode_params + +# plotting solution for x,y for chain_lux +plot(t, sol_lux.ensemblesol[1]) +plot!(t, sol_lux.ensemblesol[2]) + +# estimated weights and biases by .estimated_nn_params for chain_lux +sol_lux.estimated_nn_params + +# # ----------------------------------stats----------------------------- +# # ---------------------------- +# # ----------------------------- +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:04:47 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:03:38 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:04:12 +# # -------------------------- +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:05:09 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:04:47 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:04:25 +# # -------------- +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:06:47 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:05:54 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:05:46 +# # ------------------------ +# # ----------------------- +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -882.2934218498742 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:04:06 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -882.2934218498742 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:03:32 + +# physics Logpdf is : -15740.509286661572 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -882.2934218498742 +# L2loss2 Logpdf is : -3118.0639515039957 +# Sampling 100%|███████████████████████████████| Time: 0:03:01 +# # -------------------------- +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1411.1717435511828 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:04:02 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1411.1717435511828 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:04:08 + +# physics Logpdf is : -18864.79640643607 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -1411.1717435511828 +# L2loss2 Logpdf is : -6242.351071278482 +# Sampling 100%|███████████████████████████████| Time: 0:04:15 +# # ---------------------------- +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -3240.067149411982 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:05:37 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -3240.067149411982 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:06:02 + +# physics Logpdf is : -25119.77191296288 +# prior Logpdf is : -139.5069300318621 +# L2lossData Logpdf is : -3240.067149411982 +# L2loss2 Logpdf is : -12497.32657780532 +# Sampling 100%|███████████████████████████████| Time: 0:06:13 + +using NeuralPDE, Lux, ModelingToolkit, Optimization, OptimizationOptimJL +import ModelingToolkit: Interval, infimum, supremum + +using NeuralPDE, Flux, OptimizationOptimisers + +function diffeq(u, p, t) + u1, u2 = u + return [u2, p[1] + p[2] * sin(u1) + p[3] * u2] +end +p = [5, -10, -1.7] +u0 = [-1.0, 7.0] +tspan = (0.0, 10.0) +prob = ODEProblem(ODEFunction(diffeq), u0, tspan, p) + +chainnew = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), Flux.Dense(8, 2)) |> + Flux.f64 + +opt = OptimizationOptimisers.Adam(0.1) +opt = Optimisers.ADAGrad(0.1) +opt = Optimisers.AdaMax(0.01) +algnew = NeuralPDE.NNODE(chainnew, opt) +solution_new = solve(prob, algnew, verbose = true, + abstol = 1e-10, maxiters = 7000) +u = reduce(hcat, solution_new.u) +plot(solution_new.t, u[1, :]) +plot!(solution_new.t, u[2, :]) + +algnew = NeuralPDE.BNNODE(chainnew, draw_samples = 200, + n_leapfrog = 30, progress = true) +solution_new = solve(prob, algnew) + +@parameters t +@variables u1(..), u2(..) +D = Differential(t) +eq = [D(u1(t)) ~ u2(t), + D(u2(t)) ~ 5 - 10 * sin(u1(t)) - 1.7 * u2(t)]; + +import ModelingToolkit: Interval +bcs = [u1(0) ~ -1, u2(0) ~ 7] +domains = [t ∈ Interval(0.0, 10.0)] +dt = 0.01 + +input_ = length(domains) # number of dimensions +n = 16 +chain = [Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), Lux.Dense(n, 1)) + for _ in 1:2] + +@named pde_system = PDESystem(eq, bcs, domains, [t], [u1(t), u2(t)]) + +strategy = NeuralPDE.GridTraining(dt) +discretization = PhysicsInformedNN(chain, strategy) +sym_prob = NeuralPDE.symbolic_discretize(pde_system, discretization) + +pde_loss_functions = sym_prob.loss_functions.pde_loss_functions +bc_loss_functions = sym_prob.loss_functions.bc_loss_functions + +callback = function (p, l) + println("loss: ", l) + # println("pde_losses: ", map(l_ -> l_(p), pde_loss_functions)) + # println("bcs_losses: ", map(l_ -> l_(p), bc_loss_functions)) + return false +end + +loss_functions = [pde_loss_functions; bc_loss_functions] + +function loss_function(θ, p) + sum(map(l -> l(θ), loss_functions)) +end + +f_ = OptimizationFunction(loss_function, Optimization.AutoZygote()) +prob = Optimization.OptimizationProblem(f_, sym_prob.flat_init_params) + +res = Optimization.solve(prob, + OptimizationOptimJL.BFGS(); + callback = callback, + maxiters = 1000) +phi = discretization.phi \ No newline at end of file From a999ccffab953c47f2bfcc42e07a3bb71ee2dc36 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sat, 20 Jan 2024 23:53:20 +0530 Subject: [PATCH 06/53] removed new files --- src/BNNODE_new.jl | 794 -------- test/BPINN_newform.jl | 4354 ----------------------------------------- 2 files changed, 5148 deletions(-) delete mode 100644 src/BNNODE_new.jl delete mode 100644 test/BPINN_newform.jl diff --git a/src/BNNODE_new.jl b/src/BNNODE_new.jl deleted file mode 100644 index e6b1f24faa..0000000000 --- a/src/BNNODE_new.jl +++ /dev/null @@ -1,794 +0,0 @@ -mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, - P <: Vector{<:Distribution}, - D <: - Union{Vector{Nothing}, Vector{<:Vector{<:AbstractFloat}}}, -} - dim::Int - prob::DiffEqBase.ODEProblem - chain::C - st::S - strategy::ST - dataset::D - priors::P - phystd::Vector{Float64} - l2std::Vector{Float64} - autodiff::Bool - physdt::Float64 - extraparams::Int - init_params::I - - function LogTargetDensity(dim, prob, chain::Optimisers.Restructure, st, strategy, - dataset, - priors, phystd, l2std, autodiff, physdt, extraparams, - init_params::AbstractVector) - new{ - typeof(chain), - Nothing, - typeof(strategy), - typeof(init_params), - typeof(priors), - typeof(dataset), - }(dim, - prob, - chain, - nothing, strategy, - dataset, - priors, - phystd, - l2std, - autodiff, - physdt, - extraparams, - init_params) - end - function LogTargetDensity(dim, prob, chain::Lux.AbstractExplicitLayer, st, strategy, - dataset, - priors, phystd, l2std, autodiff, physdt, extraparams, - init_params::NamedTuple) - new{ - typeof(chain), - typeof(st), - typeof(strategy), - typeof(init_params), - typeof(priors), - typeof(dataset), - }(dim, - prob, - chain, st, strategy, - dataset, priors, - phystd, l2std, - autodiff, - physdt, - extraparams, - init_params) - end -end - -""" -cool function to convert parameter's vector to ComponentArray of parameters (for Lux Chain: vector of samples -> Lux ComponentArrays) -""" -function vector_to_parameters(ps_new::AbstractVector, ps::NamedTuple) - @assert length(ps_new) == Lux.parameterlength(ps) - i = 1 - function get_ps(x) - z = reshape(view(ps_new, i:(i + length(x) - 1)), size(x)) - i += length(x) - return z - end - return Functors.fmap(get_ps, ps) -end - -function LogDensityProblems.logdensity(Tar::LogTargetDensity, θ) - return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) - # + L2loss2(Tar, θ) -end - -LogDensityProblems.dimension(Tar::LogTargetDensity) = Tar.dim - -function LogDensityProblems.capabilities(::LogTargetDensity) - LogDensityProblems.LogDensityOrder{1}() -end - -# suggested extra loss function -function L2loss2(Tar::LogTargetDensity, θ) - f = Tar.prob.f - - # parameter estimation chosen or not - if Tar.extraparams > 0 - dataset, deri_sol = Tar.dataset - # deri_sol = deri_sol' - autodiff = Tar.autodiff - - # # Timepoints to enforce Physics - # dataset = Array(reduce(hcat, dataset)') - # t = dataset[end, :] - # û = dataset[1:(end - 1), :] - - # ode_params = Tar.extraparams == 1 ? - # θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : - # θ[((length(θ) - Tar.extraparams) + 1):length(θ)] - - # if length(û[:, 1]) == 1 - # physsol = [f(û[:, i][1], - # ode_params, - # t[i]) - # for i in 1:length(û[1, :])] - # else - # physsol = [f(û[:, i], - # ode_params, - # t[i]) - # for i in 1:length(û[1, :])] - # end - # #form of NN output matrix output dim x n - # deri_physsol = reduce(hcat, physsol) - - # > for perfect deriv(basically gradient matching in case of an ODEFunction) - # in case of PDE or general ODE we would want to reduce residue of f(du,u,p,t) - # if length(û[:, 1]) == 1 - # deri_sol = [f(û[:, i][1], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[1, :])] - # else - # deri_sol = [f(û[:, i], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[1, :])] - # end - # deri_sol = reduce(hcat, deri_sol) - # deri_sol = reduce(hcat, derivatives) - - # Timepoints to enforce Physics - t = dataset[end] - u1 = dataset[2] - û = dataset[1] - # Tar(t, θ[1:(length(θ) - Tar.extraparams)])' - # - - nnsol = NNodederi(Tar, t, θ[1:(length(θ) - Tar.extraparams)], autodiff) - - ode_params = Tar.extraparams == 1 ? - θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : - θ[((length(θ) - Tar.extraparams) + 1):length(θ)] - - if length(Tar.prob.u0) == 1 - physsol = [f(û[i], - ode_params, - t[i]) - for i in 1:length(û[:, 1])] - else - physsol = [f([û[i], u1[i]], - ode_params, - t[i]) - for i in 1:length(û[:, 1])] - end - #form of NN output matrix output dim x n - deri_physsol = reduce(hcat, physsol) - - # if length(Tar.prob.u0) == 1 - # nnsol = [f(û[i], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[:, 1])] - # else - # nnsol = [f([û[i], u1[i]], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[:, 1])] - # end - # form of NN output matrix output dim x n - # nnsol = reduce(hcat, nnsol) - - # > Instead of dataset gradients trying NN derivatives with dataset collocation - # # convert to matrix as nnsol - - physlogprob = 0 - for i in 1:length(Tar.prob.u0) - # can add phystd[i] for u[i] - physlogprob += logpdf(MvNormal(deri_physsol[i, :], - LinearAlgebra.Diagonal(map(abs2, - (Tar.l2std[i] * 4.0) .* - ones(length(nnsol[i, :]))))), - nnsol[i, :]) - end - return physlogprob - else - return 0 - end -end - -# PDE(DU,U,P,T)=0 - -# Derivated via Central Diff -# function calculate_derivatives2(dataset) -# x̂, time = dataset -# num_points = length(x̂) -# # Initialize an array to store the derivative values. -# derivatives = similar(x̂) - -# for i in 2:(num_points - 1) -# # Calculate the first-order derivative using central differences. -# Δt_forward = time[i + 1] - time[i] -# Δt_backward = time[i] - time[i - 1] - -# derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) - -# derivatives[i] = derivative -# end - -# # Derivatives at the endpoints can be calculated using forward or backward differences. -# derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) -# derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) -# return derivatives -# end - -function calderivatives(prob, dataset) - chainflux = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), - Flux.Dense(8, 2)) |> Flux.f64 - # chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 - function loss(x, y) - # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1]) + - # Flux.mse.(prob.u0[2] .+ (prob.tspan[2] .- x)' .* chainflux(x)[2, :], y[2])) - # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1])) - sum(Flux.mse.(chainflux(x), y)) - end - optimizer = Flux.Optimise.ADAM(0.01) - epochs = 3000 - for epoch in 1:epochs - Flux.train!(loss, - Flux.params(chainflux), - [(dataset[end]', dataset[1:(end - 1)])], - optimizer) - end - - # A1 = (prob.u0' .+ - # (prob.tspan[2] .- (dataset[end]' .+ sqrt(eps(eltype(Float64)))))' .* - # chainflux(dataset[end]' .+ sqrt(eps(eltype(Float64))))') - - # A2 = (prob.u0' .+ - # (prob.tspan[2] .- (dataset[end]'))' .* - # chainflux(dataset[end]')') - - A1 = chainflux(dataset[end]' .+ sqrt(eps(eltype(dataset[end][1])))) - A2 = chainflux(dataset[end]') - - gradients = (A2 .- A1) ./ sqrt(eps(eltype(dataset[end][1]))) - - return gradients -end - -function calculate_derivatives(dataset) - - # u = dataset[1] - # u1 = dataset[2] - # t = dataset[end] - # # control points - # n = Int(floor(length(t) / 10)) - # # spline for datasetvalues(solution) - # # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) - # interp = CubicSpline(u, t) - # interp1 = CubicSpline(u1, t) - # # derrivatives interpolation - # dx = t[2] - t[1] - # time = collect(t[1]:dx:t[end]) - # smoothu = [interp(i) for i in time] - # smoothu1 = [interp1(i) for i in time] - # # derivative of the spline (must match function derivative) - # û = tvdiff(smoothu, 20, 0.5, dx = dx, ε = 1) - # û1 = tvdiff(smoothu1, 20, 0.5, dx = dx, ε = 1) - # # tvdiff(smoothu, 100, 0.035, dx = dx, ε = 1) - # # FDM - # # û1 = diff(u) / dx - # # dataset[1] and smoothu are almost equal(rounding errors) - # return [û, û1] - -end - -""" -L2 loss loglikelihood(needed for ODE parameter estimation) -""" -function L2LossData(Tar::LogTargetDensity, θ) - dataset = Tar.dataset - # check if dataset is provided - if dataset isa Vector{Nothing} || Tar.extraparams == 0 - return 0 - else - # matrix(each row corresponds to vector u's rows) - nn = Tar(dataset[end], θ[1:(length(θ) - Tar.extraparams)]) - - L2logprob = 0 - for i in 1:length(Tar.prob.u0) - # for u[i] ith vector must be added to dataset,nn[1,:] is the dx in lotka_volterra - L2logprob += logpdf(MvNormal(nn[i, :], - LinearAlgebra.Diagonal(map(abs2, - (Tar.l2std[i] * 0.5) .* - ones(length(dataset[i]))))), - dataset[i]) - end - return L2logprob - end -end - -""" -physics loglikelihood over problem timespan + dataset timepoints -""" -function physloglikelihood(Tar::LogTargetDensity, θ) - f = Tar.prob.f - p = Tar.prob.p - tspan = Tar.prob.tspan - autodiff = Tar.autodiff - strategy = Tar.strategy - - # parameter estimation chosen or not - if Tar.extraparams > 0 - ode_params = Tar.extraparams == 1 ? - θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : - θ[((length(θ) - Tar.extraparams) + 1):length(θ)] - else - ode_params = p == SciMLBase.NullParameters() ? [] : p - end - - return getlogpdf(strategy, Tar, f, autodiff, tspan, ode_params, θ) -end - -function getlogpdf(strategy::GridTraining, Tar::LogTargetDensity, f, autodiff::Bool, - tspan, - ode_params, θ) - if Tar.dataset isa Vector{Nothing} - t = collect(eltype(strategy.dx), tspan[1]:(strategy.dx):tspan[2]) - else - t = vcat(collect(eltype(strategy.dx), tspan[1]:(strategy.dx):tspan[2]), - Tar.dataset[end]) - end - - sum(innerdiff(Tar, f, autodiff, t, θ, - ode_params)) -end - -function getlogpdf(strategy::StochasticTraining, - Tar::LogTargetDensity, - f, - autodiff::Bool, - tspan, - ode_params, - θ) - if Tar.dataset isa Vector{Nothing} - t = [(tspan[2] - tspan[1]) * rand() + tspan[1] for i in 1:(strategy.points)] - else - t = vcat([(tspan[2] - tspan[1]) * rand() + tspan[1] for i in 1:(strategy.points)], - Tar.dataset[end]) - end - - sum(innerdiff(Tar, f, autodiff, t, θ, - ode_params)) -end - -function getlogpdf(strategy::QuadratureTraining, Tar::LogTargetDensity, f, - autodiff::Bool, - tspan, - ode_params, θ) - function integrand(t::Number, θ) - innerdiff(Tar, f, autodiff, [t], θ, ode_params) - end - intprob = IntegralProblem(integrand, tspan[1], tspan[2], θ; nout = length(Tar.prob.u0)) - # add dataset logpdf? - sol = solve(intprob, QuadGKJL(); abstol = strategy.abstol, reltol = strategy.reltol) - sum(sol.u) -end - -function getlogpdf(strategy::WeightedIntervalTraining, Tar::LogTargetDensity, f, - autodiff::Bool, - tspan, - ode_params, θ) - minT = tspan[1] - maxT = tspan[2] - - weights = strategy.weights ./ sum(strategy.weights) - - N = length(weights) - points = strategy.points - - difference = (maxT - minT) / N - - data = Float64[] - for (index, item) in enumerate(weights) - temp_data = rand(1, trunc(Int, points * item)) .* difference .+ minT .+ - ((index - 1) * difference) - data = append!(data, temp_data) - end - - if Tar.dataset isa Vector{Nothing} - t = data - else - t = vcat(data, - Tar.dataset[end]) - end - - sum(innerdiff(Tar, f, autodiff, t, θ, - ode_params)) -end - -""" -MvNormal likelihood at each `ti` in time `t` for ODE collocation residue with NN with parameters θ -""" -function innerdiff(Tar::LogTargetDensity, f, autodiff::Bool, t::AbstractVector, θ, - ode_params) - - # Tar used for phi and LogTargetDensity object attributes access - out = Tar(t, θ[1:(length(θ) - Tar.extraparams)]) - - # # reject samples case(write clear reason why) - if any(isinf, out[:, 1]) || any(isinf, ode_params) - return -Inf - end - - # this is a vector{vector{dx,dy}}(handle case single u(float passed)) - if length(out[:, 1]) == 1 - physsol = [f(out[:, i][1], - ode_params, - t[i]) - for i in 1:length(out[1, :])] - else - physsol = [f(out[:, i], - ode_params, - t[i]) - for i in 1:length(out[1, :])] - end - physsol = reduce(hcat, physsol) - - nnsol = NNodederi(Tar, t, θ[1:(length(θ) - Tar.extraparams)], autodiff) - - vals = nnsol .- physsol - - # N dimensional vector if N outputs for NN(each row has logpdf of i[i] where u is vector of dependant variables) - return [logpdf(MvNormal(vals[i, :], - LinearAlgebra.Diagonal(map(abs2, - Tar.phystd[i] .* - ones(length(vals[i, :]))))), - zeros(length(vals[i, :]))) for i in 1:length(Tar.prob.u0)] -end - -""" -prior logpdf for NN parameters + ODE constants -""" -function priorweights(Tar::LogTargetDensity, θ) - allparams = Tar.priors - # nn weights - nnwparams = allparams[1] - - if Tar.extraparams > 0 - # Vector of ode parameters priors - invpriors = allparams[2:end] - - invlogpdf = sum(logpdf(invpriors[length(θ) - i + 1], θ[i]) - for i in (length(θ) - Tar.extraparams + 1):length(θ); init = 0.0) - - return (invlogpdf - + - logpdf(nnwparams, θ[1:(length(θ) - Tar.extraparams)])) - else - return logpdf(nnwparams, θ) - end -end - -function generate_Tar(chain::Lux.AbstractExplicitLayer, init_params) - θ, st = Lux.setup(Random.default_rng(), chain) - return init_params, chain, st -end - -function generate_Tar(chain::Lux.AbstractExplicitLayer, init_params::Nothing) - θ, st = Lux.setup(Random.default_rng(), chain) - return θ, chain, st -end - -function generate_Tar(chain::Flux.Chain, init_params) - θ, re = Flux.destructure(chain) - return init_params, re, nothing -end - -function generate_Tar(chain::Flux.Chain, init_params::Nothing) - θ, re = Flux.destructure(chain) - # find_good_stepsize,phasepoint takes only float64 - return θ, re, nothing -end - -""" -nn OUTPUT AT t,θ ~ phi(t,θ) -""" -function (f::LogTargetDensity{C, S})(t::AbstractVector, - θ) where {C <: Optimisers.Restructure, S} - f.prob.u0 .+ (t' .- f.prob.tspan[1]) .* f.chain(θ)(adapt(parameterless_type(θ), t')) -end - -function (f::LogTargetDensity{C, S})(t::AbstractVector, - θ) where {C <: Lux.AbstractExplicitLayer, S} - θ = vector_to_parameters(θ, f.init_params) - y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), t'), θ, f.st) - ChainRulesCore.@ignore_derivatives f.st = st - f.prob.u0 .+ (t' .- f.prob.tspan[1]) .* y -end - -function (f::LogTargetDensity{C, S})(t::Number, - θ) where {C <: Optimisers.Restructure, S} - # must handle paired odes hence u0 broadcasted - f.prob.u0 .+ (t - f.prob.tspan[1]) * f.chain(θ)(adapt(parameterless_type(θ), [t])) -end - -function (f::LogTargetDensity{C, S})(t::Number, - θ) where {C <: Lux.AbstractExplicitLayer, S} - θ = vector_to_parameters(θ, f.init_params) - y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), [t]), θ, f.st) - ChainRulesCore.@ignore_derivatives f.st = st - f.prob.u0 .+ (t .- f.prob.tspan[1]) .* y -end - -""" -similar to ode_dfdx() in NNODE/ode_solve.jl -""" -function NNodederi(phi::LogTargetDensity, t::AbstractVector, θ, autodiff::Bool) - if autodiff - hcat(ForwardDiff.derivative.(ti -> phi(ti, θ), t)...) - else - (phi(t .+ sqrt(eps(eltype(t))), θ) - phi(t, θ)) ./ sqrt(eps(eltype(t))) - end -end - -function kernelchoice(Kernel, max_depth, Δ_max, n_leapfrog, δ, λ) - if Kernel == HMC - Kernel(n_leapfrog) - elseif Kernel == HMCDA - Kernel(δ, λ) - else - Kernel(δ, max_depth = max_depth, Δ_max = Δ_max) - end -end - -function integratorchoice(Integrator, initial_ϵ, jitter_rate, - tempering_rate) - if Integrator == JitteredLeapfrog - Integrator(initial_ϵ, jitter_rate) - elseif Integrator == TemperedLeapfrog - Integrator(initial_ϵ, tempering_rate) - else - Integrator(initial_ϵ) - end -end - -function adaptorchoice(Adaptor, mma, ssa) - if Adaptor != AdvancedHMC.NoAdaptation() - Adaptor(mma, ssa) - else - AdvancedHMC.NoAdaptation() - end -end - -""" -```julia -ahmc_bayesian_pinn_ode(prob, chain; strategy = GridTraining, - dataset = [nothing],init_params = nothing, - draw_samples = 1000, physdt = 1 / 20.0f0,l2std = [0.05], - phystd = [0.05], priorsNNw = (0.0, 2.0), - param = [],nchains = 1,autodiff = false, Kernel = HMC, - Integrator = Leapfrog, Adaptor = StanHMCAdaptor, - targetacceptancerate = 0.8, Metric = DiagEuclideanMetric, - jitter_rate = 3.0, tempering_rate = 3.0, max_depth = 10, - Δ_max = 1000, n_leapfrog = 10, δ = 0.65, λ = 0.3, - progress = false,verbose = false) -``` -!!! warn - - Note that ahmc_bayesian_pinn_ode() only supports ODEs which are written in the out-of-place form, i.e. - `du = f(u,p,t)`, and not `f(du,u,p,t)`. If not declared out-of-place, then the ahmc_bayesian_pinn_ode() - will exit with an error. - -## Example -linear = (u, p, t) -> -u / p[1] + exp(t / p[2]) * cos(t) -tspan = (0.0, 10.0) -u0 = 0.0 -p = [5.0, -5.0] -prob = ODEProblem(linear, u0, tspan, p) - -# CREATE DATASET (Necessity for accurate Parameter estimation) -sol = solve(prob, Tsit5(); saveat = 0.05) -u = sol.u[1:100] -time = sol.t[1:100] - -# dataset and BPINN create -x̂ = collect(Float64, Array(u) + 0.05 * randn(size(u))) -dataset = [x̂, time] - -chainflux1 = Flux.Chain(Flux.Dense(1, 5, tanh), Flux.Dense(5, 5, tanh), Flux.Dense(5, 1) - -# simply solving ode here hence better to not pass dataset(uses ode params specified in prob) -fh_mcmc_chainflux1, fhsamplesflux1, fhstatsflux1 = ahmc_bayesian_pinn_ode(prob,chainflux1, - dataset = dataset, - draw_samples = 1500, - l2std = [0.05], - phystd = [0.05], - priorsNNw = (0.0,3.0)) - -# solving ode + estimating parameters hence dataset needed to optimize parameters upon + Pior Distributions for ODE params -fh_mcmc_chainflux2, fhsamplesflux2, fhstatsflux2 = ahmc_bayesian_pinn_ode(prob,chainflux1, - dataset = dataset, - draw_samples = 1500, - l2std = [0.05], - phystd = [0.05], - priorsNNw = (0.0,3.0), - param = [Normal(6.5,0.5),Normal(-3,0.5)]) - -## NOTES -Dataset is required for accurate Parameter estimation + solving equations -Incase you are only solving the Equations for solution, do not provide dataset - -## Positional Arguments -* `prob`: DEProblem(out of place and the function signature should be f(u,p,t) -* `chain`: Lux/Flux Neural Netork which would be made the Bayesian PINN - -## Keyword Arguments -* `strategy`: The training strategy used to choose the points for the evaluations. By default GridTraining is used with given physdt discretization. -* `dataset`: Vector containing Vectors of corresponding u,t values -* `init_params`: intial parameter values for BPINN (ideally for multiple chains different initializations preferred) -* `nchains`: number of chains you want to sample (random initialisation of params by default) -* `draw_samples`: number of samples to be drawn in the MCMC algorithms (warmup samples are ~2/3 of draw samples) -* `l2std`: standard deviation of BPINN predicition against L2 losses/Dataset -* `phystd`: standard deviation of BPINN predicition against Chosen Underlying ODE System -* `priorsNNw`: Vector of [mean, std] for BPINN parameter. Weights and Biases of BPINN are Normal Distributions by default -* `param`: Vector of chosen ODE parameters Distributions in case of Inverse problems. -* `autodiff`: Boolean Value for choice of Derivative Backend(default is numerical) -* `physdt`: Timestep for approximating ODE in it's Time domain. (1/20.0 by default) - -# AHMC.jl is still developing convenience structs so might need changes on new releases. -* `Kernel`: Choice of MCMC Sampling Algorithm (AdvancedHMC.jl implemenations HMC/NUTS/HMCDA) -* `targetacceptancerate`: Target percentage(in decimal) of iterations in which the proposals were accepted(0.8 by default) -* `Integrator(jitter_rate, tempering_rate), Metric, Adaptor`: https://turinglang.org/AdvancedHMC.jl/stable/ -* `max_depth`: Maximum doubling tree depth (NUTS) -* `Δ_max`: Maximum divergence during doubling tree (NUTS) -* `n_leapfrog`: number of leapfrog steps for HMC -* `δ`: target acceptance probability for NUTS/HMCDA -* `λ`: target trajectory length for HMCDA -* `progress`: controls whether to show the progress meter or not. -* `verbose`: controls the verbosity. (Sample call args in AHMC) - -""" - -""" -dataset would be (x̂,t) -priors: pdf for W,b + pdf for ODE params -""" -function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; - strategy = GridTraining, dataset = [nothing], - init_params = nothing, draw_samples = 1000, - physdt = 1 / 20.0, l2std = [0.05], - phystd = [0.05], priorsNNw = (0.0, 2.0), - param = [], nchains = 1, autodiff = false, - Kernel = HMC, Integrator = Leapfrog, - Adaptor = StanHMCAdaptor, targetacceptancerate = 0.8, - Metric = DiagEuclideanMetric, jitter_rate = 3.0, - tempering_rate = 3.0, max_depth = 10, Δ_max = 1000, - n_leapfrog = 10, δ = 0.65, λ = 0.3, progress = false, - verbose = false) - - # NN parameter prior mean and variance(PriorsNN must be a tuple) - if isinplace(prob) - throw(error("The BPINN ODE solver only supports out-of-place ODE definitions, i.e. du=f(u,p,t).")) - end - - strategy = strategy == GridTraining ? strategy(physdt) : strategy - - if dataset != [nothing] && - (length(dataset) < 2 || !(typeof(dataset) <: Vector{<:Vector{<:AbstractFloat}})) - throw(error("Invalid dataset. dataset would be timeseries (x̂,t) where type: Vector{Vector{AbstractFloat}")) - end - - if dataset != [nothing] && param == [] - println("Dataset is only needed for Parameter Estimation + Forward Problem, not in only Forward Problem case.") - elseif dataset == [nothing] && param != [] - throw(error("Dataset Required for Parameter Estimation.")) - end - - if chain isa Lux.AbstractExplicitLayer || chain isa Flux.Chain - # Flux-vector, Lux-Named Tuple - initial_nnθ, recon, st = generate_Tar(chain, init_params) - else - error("Only Lux.AbstractExplicitLayer and Flux.Chain neural networks are supported") - end - - if nchains > Threads.nthreads() - throw(error("number of chains is greater than available threads")) - elseif nchains < 1 - throw(error("number of chains must be greater than 1")) - end - - # eltype(physdt) cause needs Float64 for find_good_stepsize - if chain isa Lux.AbstractExplicitLayer - # Lux chain(using component array later as vector_to_parameter need namedtuple) - initial_θ = collect(eltype(physdt), - vcat(ComponentArrays.ComponentArray(initial_nnθ))) - else - initial_θ = collect(eltype(physdt), initial_nnθ) - end - - # adding ode parameter estimation - nparameters = length(initial_θ) - ninv = length(param) - priors = [ - MvNormal(priorsNNw[1] * ones(nparameters), - LinearAlgebra.Diagonal(map(abs2, priorsNNw[2] .* ones(nparameters)))), - ] - - # append Ode params to all paramvector - if ninv > 0 - # shift ode params(initialise ode params by prior means) - initial_θ = vcat(initial_θ, [Distributions.params(param[i])[1] for i in 1:ninv]) - priors = vcat(priors, param) - nparameters += ninv - end - - t0 = prob.tspan[1] - # dimensions would be total no of params,initial_nnθ for Lux namedTuples - ℓπ = LogTargetDensity(nparameters, prob, recon, st, strategy, dataset, priors, - phystd, l2std, autodiff, physdt, ninv, initial_nnθ) - - try - ℓπ(t0, initial_θ[1:(nparameters - ninv)]) - catch err - if isa(err, DimensionMismatch) - throw(DimensionMismatch("Dimensions of the initial u0 and chain should match")) - else - throw(err) - end - end - - # Define Hamiltonian system (nparameters ~ dimensionality of the sampling space) - metric = Metric(nparameters) - hamiltonian = Hamiltonian(metric, ℓπ, ForwardDiff) - - println("physics Logpdf is : ", physloglikelihood(ℓπ, initial_θ)) - println("prior Logpdf is : ", priorweights(ℓπ, initial_θ)) - println("L2lossData Logpdf is : ", L2LossData(ℓπ, initial_θ)) - println("L2loss2 Logpdf is : ", L2loss2(ℓπ, initial_θ)) - - # parallel sampling option - if nchains != 1 - # Cache to store the chains - chains = Vector{Any}(undef, nchains) - statsc = Vector{Any}(undef, nchains) - samplesc = Vector{Any}(undef, nchains) - - Threads.@threads for i in 1:nchains - # each chain has different initial NNparameter values(better posterior exploration) - initial_θ = vcat(randn(nparameters - ninv), - initial_θ[(nparameters - ninv + 1):end]) - initial_ϵ = find_good_stepsize(hamiltonian, initial_θ) - integrator = integratorchoice(Integrator, initial_ϵ, jitter_rate, - tempering_rate) - adaptor = adaptorchoice(Adaptor, MassMatrixAdaptor(metric), - StepSizeAdaptor(targetacceptancerate, integrator)) - Kernel = AdvancedHMC.make_kernel(kernelchoice(Kernel, max_depth, Δ_max, - n_leapfrog, δ, λ), integrator) - samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, adaptor; - progress = progress, verbose = verbose) - - samplesc[i] = samples - statsc[i] = stats - mcmc_chain = Chains(hcat(samples...)') - chains[i] = mcmc_chain - end - - return chains, samplesc, statsc - else - initial_ϵ = find_good_stepsize(hamiltonian, initial_θ) - integrator = integratorchoice(Integrator, initial_ϵ, jitter_rate, tempering_rate) - adaptor = adaptorchoice(Adaptor, MassMatrixAdaptor(metric), - StepSizeAdaptor(targetacceptancerate, integrator)) - Kernel = AdvancedHMC.make_kernel(kernelchoice(Kernel, max_depth, Δ_max, n_leapfrog, - δ, λ), integrator) - samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, - adaptor; progress = progress, verbose = verbose) - - # return a chain(basic chain),samples and stats - matrix_samples = hcat(samples...) - mcmc_chain = MCMCChains.Chains(matrix_samples') - return mcmc_chain, samples, stats - end -end \ No newline at end of file diff --git a/test/BPINN_newform.jl b/test/BPINN_newform.jl deleted file mode 100644 index fa2f04073e..0000000000 --- a/test/BPINN_newform.jl +++ /dev/null @@ -1,4354 +0,0 @@ -# # Testing Code -using Test, MCMCChains -using ForwardDiff, Distributions, OrdinaryDiffEq -using Flux, OptimizationOptimisers, AdvancedHMC, Lux -using Statistics, Random, Functors, ComponentArrays -using NeuralPDE, MonteCarloMeasurements - -# note that current testing bounds can be easily further tightened but have been inflated for support for Julia build v1 -# on latest Julia version it performs much better for below tests -Random.seed!(100) - -# for sampled params->lux ComponentArray -function vector_to_parameters(ps_new::AbstractVector, ps::NamedTuple) - @assert length(ps_new) == Lux.parameterlength(ps) - i = 1 - function get_ps(x) - z = reshape(view(ps_new, i:(i + length(x) - 1)), size(x)) - i += length(x) - return z - end - return Functors.fmap(get_ps, ps) -end - -## PROBLEM-1 (WITHOUT PARAMETER ESTIMATION) -linear_analytic = (u0, p, t) -> u0 + sin(2 * π * t) / (2 * π) -linear = (u, p, t) -> cos(2 * π * t) -tspan = (0.0, 2.0) -u0 = 0.0 -prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan) -p = prob.p - -# Numerical and Analytical Solutions: testing ahmc_bayesian_pinn_ode() -ta = range(tspan[1], tspan[2], length = 300) -u = [linear_analytic(u0, nothing, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.02 * randn(size(u))) -time = vec(collect(Float64, ta)) -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# testing points for solve() call must match saveat(1/50.0) arg -ta0 = range(tspan[1], tspan[2], length = 101) -u1 = [linear_analytic(u0, nothing, ti) for ti in ta0] -x̂1 = collect(Float64, Array(u1) + 0.02 * randn(size(u1))) -time1 = vec(collect(Float64, ta0)) -physsol0_1 = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] - -chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 -chainlux = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) -init1, re1 = destructure(chainflux) -θinit, st = Lux.setup(Random.default_rng(), chainlux) - -fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainflux, - draw_samples = 2500, - n_leapfrog = 30) - -fh_mcmc_chain2, fhsamples2, fhstats2 = ahmc_bayesian_pinn_ode(prob, chainlux, - draw_samples = 2500, - n_leapfrog = 30) - -# can change training strategies by adding this to call (Quadratuer and GridTraining show good results but stochastics sampling techniques perform bad) -# strategy = QuadratureTraining(; quadrature_alg = QuadGKJL(), -# reltol = 1e-6, -# abstol = 1e-3, maxiters = 1000, -# batch = 0) - -alg = NeuralPDE.BNNODE(chainflux, draw_samples = 2500, - n_leapfrog = 30) -sol1flux = solve(prob, alg) - -alg = NeuralPDE.BNNODE(chainlux, draw_samples = 2500, - n_leapfrog = 30) -sol1lux = solve(prob, alg) - -# testing points -t = time -# Mean of last 500 sampled parameter's curves(flux and lux chains)[Ensemble predictions] -out = re1.(fhsamples1[(end - 500):end]) -yu = collect(out[i](t') for i in eachindex(out)) -fluxmean = [mean(vcat(yu...)[:, i]) for i in eachindex(t)] -meanscurve1 = prob.u0 .+ (t .- prob.tspan[1]) .* fluxmean - -θ = [vector_to_parameters(fhsamples1[i], θinit) for i in 2000:2500] -luxar = [chainlux(t', θ[i], st)[1] for i in 1:500] -luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] -meanscurve2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean - -# --------------------- ahmc_bayesian_pinn_ode() call -@test mean(abs.(x̂ .- meanscurve1)) < 0.05 -@test mean(abs.(physsol1 .- meanscurve1)) < 0.005 -@test mean(abs.(x̂ .- meanscurve2)) < 0.05 -@test mean(abs.(physsol1 .- meanscurve2)) < 0.005 - -#--------------------- solve() call -@test mean(abs.(x̂1 .- sol1flux.ensemblesol[1])) < 0.05 -@test mean(abs.(physsol0_1 .- sol1flux.ensemblesol[1])) < 0.05 -@test mean(abs.(x̂1 .- sol1lux.ensemblesol[1])) < 0.05 -@test mean(abs.(physsol0_1 .- sol1lux.ensemblesol[1])) < 0.05 - -## PROBLEM-1 (WITH PARAMETER ESTIMATION) -linear_analytic = (u0, p, t) -> u0 + sin(p * t) / (p) -linear = (u, p, t) -> cos(p * t) -tspan = (0.0, 2.0) -u0 = 0.0 -p = 2 * pi -prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan, p) - -# Numerical and Analytical Solutions -sol1 = solve(prob, Tsit5(); saveat = 0.01) -u = sol1.u -time = sol1.t - -# BPINN AND TRAINING DATASET CREATION(dataset must be defined only inside problem timespan!) -ta = range(tspan[1], tspan[2], length = 25) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) .+ (0.2 .* Array(u) .* randn(size(u)))) -time = vec(collect(Float64, ta)) -dataset = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# testing points for solve call(saveat=1/50.0 ∴ at t = collect(eltype(saveat), prob.tspan[1]:saveat:prob.tspan[2] internally estimates) -ta0 = range(tspan[1], tspan[2], length = 101) -u1 = [linear_analytic(u0, p, ti) for ti in ta0] -x̂1 = collect(Float64, Array(u1) + 0.2 * randn(size(u1))) -time1 = vec(collect(Float64, ta0)) -physsol1_1 = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] - -using Plots, StatsPlots -# plot(dataset[2], calderivatives(dataset)') -yu = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) -plot(yu, [linear_analytic(u0, p, t) for t in yu]) -chainflux1 = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 -chainlux1 = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) -init1, re1 = destructure(chainflux1) -θinit, st = Lux.setup(Random.default_rng(), chainlux1) - -fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainflux1, - dataset = dataset, - draw_samples = 2500, - physdt = 1 / 50.0f0, - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(9, - 0.5), - ], - Metric = DiagEuclideanMetric, - n_leapfrog = 30) - -fh_mcmc_chain2, fhsamples2, fhstats2 = ahmc_bayesian_pinn_ode(prob, chainlux1, - dataset = dataset, - draw_samples = 2500, - physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30) - -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 10.0), - l2std = [0.005], phystd = [0.01], - param = [Normal(11, 6)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30) -# original paper (pure data 0 1) -sol1flux = solve(prob, alg) -sol1flux.estimated_ode_params -# pure data method 1 1 -sol2flux = solve(prob, alg) -sol2flux.estimated_ode_params -# pure data method 1 0 -sol3flux = solve(prob, alg) -sol3flux.estimated_ode_params -# deri collocation -sol4flux = solve(prob, alg) -sol4flux.estimated_ode_params -# collocation -sol5flux = solve(prob, alg) -sol5flux.estimated_ode_params -# collocation + L2Data loss(at 9,0.5 1,2 gives same) -sol6flux = solve(prob, alg) -sol6flux.estimated_ode_params -# 2500 iters -sol7flux = solve(prob, alg) -sol7flux.estimated_ode_params - -plotly() -plot!(yu, sol1flux.ensemblesol[1]) -plot!(yu, sol2flux.ensemblesol[1]) -plot!(yu, sol3flux.ensemblesol[1]) -plot!(yu, sol4flux.ensemblesol[1]) -plot!(yu, sol5flux.ensemblesol[1]) -plot!(yu, sol6flux.ensemblesol[1]) - -plot!(dataset[2], dataset[1]) - -# plot!(sol4flux.ensemblesol[1]) -# plot!(sol5flux.ensemblesol[1]) - -sol2flux.estimated_ode_params - -sol1flux.estimated_ode_params - -sol3flux.estimated_ode_params - -sol4flux.estimated_ode_params - -sol5flux.estimated_ode_params - -alg = NeuralPDE.BNNODE(chainlux1, dataset = dataset, - draw_samples = 2500, - physdt = 1 / 50.0f0, - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(9, - 0.5), - ], - Metric = DiagEuclideanMetric, - n_leapfrog = 30) - -sol2lux = solve(prob, alg) - -# testing points -t = time -# Mean of last 500 sampled parameter's curves(flux and lux chains)[Ensemble predictions] -out = re1.([fhsamples1[i][1:22] for i in 2000:2500]) -yu = collect(out[i](t') for i in eachindex(out)) -fluxmean = [mean(vcat(yu...)[:, i]) for i in eachindex(t)] -meanscurve1 = prob.u0 .+ (t .- prob.tspan[1]) .* fluxmean - -θ = [vector_to_parameters(fhsamples2[i][1:(end - 1)], θinit) for i in 2000:2500] -luxar = [chainlux1(t', θ[i], st)[1] for i in 1:500] -luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] -meanscurve2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean - -# --------------------- ahmc_bayesian_pinn_ode() call -@test mean(abs.(physsol1 .- meanscurve1)) < 0.15 -@test mean(abs.(physsol1 .- meanscurve2)) < 0.15 - -# ESTIMATED ODE PARAMETERS (NN1 AND NN2) -@test abs(p - mean([fhsamples2[i][23] for i in 2000:2500])) < abs(0.25 * p) -@test abs(p - mean([fhsamples1[i][23] for i in 2000:2500])) < abs(0.25 * p) - -#-------------------------- solve() call -@test mean(abs.(physsol1_1 .- sol2flux.ensemblesol[1])) < 8e-2 -@test mean(abs.(physsol1_1 .- sol2lux.ensemblesol[1])) < 8e-2 - -# ESTIMATED ODE PARAMETERS (NN1 AND NN2) -@test abs(p - sol1flux.estimated_ode_params[1]) < abs(0.15 * p) -@test abs(p - sol2lux.estimated_ode_params[1]) < abs(0.15 * p) - -## PROBLEM-2 -linear = (u, p, t) -> u / p + exp(t / p) * cos(t) -tspan = (0.0, 10.0) -u0 = 0.0 -p = -5.0 -prob = ODEProblem(linear, u0, tspan, p) -linear_analytic = (u0, p, t) -> exp(t / p) * (u0 + sin(t)) - -# SOLUTION AND CREATE DATASET -sol = solve(prob, Tsit5(); saveat = 0.1) -u = sol.u -time = sol.t -x̂ = u .+ (u .* 0.2) .* randn(size(u)) -dataset = [x̂, time] -t = sol.t -physsol1 = [linear_analytic(prob.u0, p, t[i]) for i in eachindex(t)] - -ta0 = range(tspan[1], tspan[2], length = 501) -u1 = [linear_analytic(u0, p, ti) for ti in ta0] -time1 = vec(collect(Float64, ta0)) -physsol2 = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] - -chainflux12 = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), - Flux.Dense(6, 1)) |> Flux.f64 -chainlux12 = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), Lux.Dense(6, 1)) -init1, re1 = destructure(chainflux12) -θinit, st = Lux.setup(Random.default_rng(), chainlux12) - -using Flux -using Random - -function derivatives(chainflux, dataset) - loss(x, y) = Flux.mse(chainflux(x), y) - optimizer = Flux.Optimise.ADAM(0.01) - epochs = 2500 - for epoch in 1:epochs - Flux.train!(loss, Flux.params(chainflux), [(dataset[2]', dataset[1]')], optimizer) - end - getgradient(chainflux, dataset) -end - -function getgradient(chainflux, dataset) - return (chainflux(dataset[end]' .+ sqrt(eps(eltype(Float64)))) .- - chainflux(dataset[end]')) ./ - sqrt(eps(eltype(dataset[end][1]))) -end - -ans = derivatives(chainflux12, dataset) - -init3, re = destructure(chainflux12) -init2 == init1 -init3 == init2 -plot!(dataset[end], ans') -plot!(dataset[end], chainflux12(dataset[end]')') - -ars = getgradient(chainflux12, dataset) - -plot!(dataset[end], ars') - -fh_mcmc_chainflux12, fhsamplesflux12, fhstatsflux12 = ahmc_bayesian_pinn_ode(prob, - chainflux12, - draw_samples = 1500, - l2std = [0.03], - phystd = [ - 0.03], - priorsNNw = (0.0, - 10.0), - n_leapfrog = 30) - -fh_mcmc_chainflux22, fhsamplesflux22, fhstatsflux22 = ahmc_bayesian_pinn_ode(prob, - chainflux12, - dataset = dataset, - draw_samples = 1500, - l2std = [0.03], - phystd = [ - 0.03, - ], - priorsNNw = (0.0, - 10.0), - param = [ - Normal(-7, - 4), - ], - n_leapfrog = 30) - -fh_mcmc_chainlux12, fhsampleslux12, fhstatslux12 = ahmc_bayesian_pinn_ode(prob, chainlux12, - draw_samples = 1500, - l2std = [0.03], - phystd = [0.03], - priorsNNw = (0.0, - 10.0), - n_leapfrog = 30) - -fh_mcmc_chainlux22, fhsampleslux22, fhstatslux22 = ahmc_bayesian_pinn_ode(prob, chainlux12, - dataset = dataset, - draw_samples = 1500, - l2std = [0.03], - phystd = [0.03], - priorsNNw = (0.0, - 10.0), - param = [ - Normal(-7, - 4), - ], - n_leapfrog = 30) - -alg1 = NeuralPDE.BNNODE(chainflux12, - dataset = dataset, - draw_samples = 500, - l2std = [0.01], - phystd = [ - 0.03, - ], - priorsNNw = (0.0, - 10.0), - param = [ - Normal(-7, - 4), - ], - n_leapfrog = 30, progress = true) - -# original paper (pure data 0 1) -sol1flux_pestim = solve(prob, alg1) -sol1flux_pestim.estimated_ode_params -# pure data method 1 1 -sol2flux_pestim = solve(prob, alg1) -sol2flux_pestim.estimated_ode_params -# pure data method 1 0 -sol3flux_pestim = solve(prob, alg1) -sol3flux_pestim.estimated_ode_params -# deri collocation -sol4flux_pestim = solve(prob, alg1) -sol4flux_pestim.estimated_ode_params -# collocation -sol5flux_pestim = solve(prob, alg1) -sol5flux_pestim.estimated_ode_params -# collocation + L2Data loss(at 9,0.5 1,2 gives same) -sol6flux_pestim = solve(prob, alg1) -sol6flux_pestim.estimated_ode_params - -using Plots, StatsPlots -ars = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) -plot(time, u) -plot!(ars, sol1flux_pestim.ensemblesol[1]) -plot!(ars, sol2flux_pestim.ensemblesol[1]) -plot!(ars, sol3flux_pestim.ensemblesol[1]) -plot!(ars, sol4flux_pestim.ensemblesol[1]) -plot!(ars, sol5flux_pestim.ensemblesol[1]) -plot!(ars, sol6flux_pestim.ensemblesol[1]) - -sol3flux_pestim.estimated_ode_params - -sol4flux_pestim.estimated_ode_params - -sol5flux_pestim.estimated_ode_params - -sol6flux_pestim.estimated_ode_params - -ars = collect(prob.tspan[1]:(1 / 50.0):prob.tspan[2]) - -init, re1 = destructure(chainflux12) -init -init1 -alg = NeuralPDE.BNNODE(chainlux12, - dataset = dataset, - draw_samples = 1500, - l2std = [0.03], - phystd = [0.03], - priorsNNw = (0.0, - 10.0), - param = [ - Normal(-7, - 4), - ], - n_leapfrog = 30) - -sol3lux_pestim = solve(prob, alg) - -# testing timepoints -t = sol.t -#------------------------------ ahmc_bayesian_pinn_ode() call -# Mean of last 500 sampled parameter's curves(flux chains)[Ensemble predictions] -out = re1.([fhsamplesflux12[i][1:61] for i in 1000:1500]) -yu = [out[i](t') for i in eachindex(out)] -fluxmean = [mean(vcat(yu...)[:, i]) for i in eachindex(t)] -meanscurve1_1 = prob.u0 .+ (t .- prob.tspan[1]) .* fluxmean - -out = re1.([fhsamplesflux22[i][1:61] for i in 1000:1500]) -yu = [out[i](t') for i in eachindex(out)] -fluxmean = [mean(vcat(yu...)[:, i]) for i in eachindex(t)] -meanscurve1_2 = prob.u0 .+ (t .- prob.tspan[1]) .* fluxmean - -@test mean(abs.(sol.u .- meanscurve1_1)) < 1e-2 -@test mean(abs.(physsol1 .- meanscurve1_1)) < 1e-2 -@test mean(abs.(sol.u .- meanscurve1_2)) < 5e-2 -@test mean(abs.(physsol1 .- meanscurve1_2)) < 5e-2 - -# estimated parameters(flux chain) -param1 = mean(i[62] for i in fhsamplesflux22[1000:1500]) -@test abs(param1 - p) < abs(0.3 * p) - -# Mean of last 500 sampled parameter's curves(lux chains)[Ensemble predictions] -θ = [vector_to_parameters(fhsampleslux12[i], θinit) for i in 1000:1500] -luxar = [chainlux12(t', θ[i], st)[1] for i in 1:500] -luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] -meanscurve2_1 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean - -θ = [vector_to_parameters(fhsampleslux22[i][1:(end - 1)], θinit) for i in 1000:1500] -luxar = [chainlux12(t', θ[i], st)[1] for i in 1:500] -luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] -meanscurve2_2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean - -@test mean(abs.(sol.u .- meanscurve2_1)) < 1e-1 -@test mean(abs.(physsol1 .- meanscurve2_1)) < 1e-1 -@test mean(abs.(sol.u .- meanscurve2_2)) < 5e-2 -@test mean(abs.(physsol1 .- meanscurve2_2)) < 5e-2 - -# estimated parameters(lux chain) -param1 = mean(i[62] for i in fhsampleslux22[1000:1500]) -@test abs(param1 - p) < abs(0.3 * p) - -#-------------------------- solve() call -# (flux chain) -@test mean(abs.(physsol2 .- sol3flux_pestim.ensemblesol[1])) < 0.15 -# estimated parameters(flux chain) -param1 = sol3flux_pestim.estimated_ode_params[1] -@test abs(param1 - p) < abs(0.45 * p) - -# (lux chain) -@test mean(abs.(physsol2 .- sol3lux_pestim.ensemblesol[1])) < 0.15 -# estimated parameters(lux chain) -param1 = sol3lux_pestim.estimated_ode_params[1] -@test abs(param1 - p) < abs(0.45 * p) - -using Plots, StatsPlots -using NoiseRobustDifferentiation, Weave, DataInterpolations - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood -# # 25 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, -# draw_samples = 1500, physdt = 1 / 50.0f0, phystd = [0.01], -# l2std = [0.01], -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux1 = solve(prob, alg) -# sol2flux1.estimated_ode_params[1] #6.41722 Particles{Float64, 1}, 6.02404 Particles{Float64, 1} -# sol2flux2 = solve(prob, alg) -# sol2flux2.estimated_ode_params[1] #6.42782 Particles{Float64, 1}, 6.07509 Particles{Float64, 1} -# sol2flux3 = solve(prob, alg) -# sol2flux3.estimated_ode_params[1] #6.42782 Particles{Float64, 1}, 6.00825 Particles{Float64, 1} - -# # 50 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux11 = solve(prob, alg) -# sol2flux11.estimated_ode_params[1] #5.71268 Particles{Float64, 1}, 6.07242 Particles{Float64, 1} -# sol2flux22 = solve(prob, alg) -# sol2flux22.estimated_ode_params[1] #5.74599 Particles{Float64, 1}, 6.04837 Particles{Float64, 1} -# sol2flux33 = solve(prob, alg) -# sol2flux33.estimated_ode_params[1] #5.74599 Particles{Float64, 1}, 6.02838 Particles{Float64, 1} - -# # 100 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux111 = solve(prob, alg) -# sol2flux111.estimated_ode_params[1] #6.59097 Particles{Float64, 1}, 5.89384 Particles{Float64, 1} -# sol2flux222 = solve(prob, alg) -# sol2flux222.estimated_ode_params[1] #6.62813 Particles{Float64, 1}, 5.88216 Particles{Float64, 1} -# sol2flux333 = solve(prob, alg) -# sol2flux333.estimated_ode_params[1] #6.62813 Particles{Float64, 1}, 5.85327 Particles{Float64, 1} - -# # ---------------------------------------------------------- -# # physdt=1/20, full likelihood cdm -# # 25 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux1_cdm = solve(prob, alg) -# sol2flux1_cdm.estimated_ode_params[1]# 6.50506 Particles{Float64, 1} ,6.38963 Particles{Float64, 1} -# sol2flux2_cdm = solve(prob, alg) -# sol2flux2_cdm.estimated_ode_params[1] #6.50032 Particles{Float64, 1} ,6.39817 Particles{Float64, 1} -# sol2flux3_cdm = solve(prob, alg) -# sol2flux3_cdm.estimated_ode_params[1] #6.50032 Particles{Float64, 1} ,6.36296 Particles{Float64, 1} - -# # 50 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux11_cdm = solve(prob, alg) -# sol2flux11_cdm.estimated_ode_params[1] #6.52951 Particles{Float64, 1},5.15621 Particles{Float64, 1} -# sol2flux22_cdm = solve(prob, alg) -# sol2flux22_cdm.estimated_ode_params[1] #6.54988 Particles{Float64, 1},5.16363 Particles{Float64, 1} -# sol2flux33_cdm = solve(prob, alg) -# sol2flux33_cdm.estimated_ode_params[1] #6.54988 Particles{Float64, 1},5.15591 Particles{Float64, 1} - -# # 100 points -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux111_cdm = solve(prob, alg) -# sol2flux111_cdm.estimated_ode_params[1] #6.74338 Particles{Float64, 1}, 9.72422 Particles{Float64, 1} -# sol2flux222_cdm = solve(prob, alg) -# sol2flux222_cdm.estimated_ode_params[1] #6.72642 Particles{Float64, 1}, 9.71991 Particles{Float64, 1} -# sol2flux333_cdm = solve(prob, alg) -# sol2flux333_cdm.estimated_ode_params[1] #6.72642 Particles{Float64, 1}, 9.75045 Particles{Float64, 1} - -# -------------------------------------------------------------------------------------- -# NEW SERIES OF TESTS (IN ORDER OF EXECUTION) -# ------------------------------------------------------------------------------------- -# original paper implementaion -# 25 points -ta = range(tspan[1], tspan[2], length = 25) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, u .+ 0.05 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset1 = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] -# scatter!(time, u) -# dataset -# scatter!(dataset1[2], dataset1[1]) -# plot(time, physsol1) - -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_normal = solve(prob, alg) -sol2flux1_normal.estimated_ode_params[1] #7.70593 Particles{Float64, 1}, 6.36096 Particles{Float64, 1} | 6.45865 Particles{Float64, 1} -sol2flux2_normal = solve(prob, alg) -sol2flux2_normal.estimated_ode_params[1] #6.66347 Particles{Float64, 1}, 6.36974 Particles{Float64, 1} | 6.45865 Particles{Float64, 1} -sol2flux3_normal = solve(prob, alg) -sol2flux3_normal.estimated_ode_params[1] #6.84827 Particles{Float64, 1}, 6.29555 Particles{Float64, 1} | 6.39947 Particles{Float64, 1} - -# 50 points -ta = range(tspan[1], tspan[2], length = 50) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.05 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset2 = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_normal = solve(prob, alg) -sol2flux11_normal.estimated_ode_params[1] #7.83577 Particles{Float64, 1},6.24652 Particles{Float64, 1} | 6.34495 Particles{Float64, 1} -sol2flux22_normal = solve(prob, alg) -sol2flux22_normal.estimated_ode_params[1] #6.49477 Particles{Float64, 1},6.2118 Particles{Float64, 1} | 6.32476 Particles{Float64, 1} -sol2flux33_normal = solve(prob, alg) -sol2flux33_normal.estimated_ode_params[1] #6.47421 Particles{Float64, 1},6.33687 Particles{Float64, 1} | 6.2448 Particles{Float64, 1} - -# 100 points -ta = range(tspan[1], tspan[2], length = 100) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.05 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset3 = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_normal = solve(prob, alg) -sol2flux111_normal.estimated_ode_params[1] #5.96604 Particles{Float64, 1},5.99588 Particles{Float64, 1} | 6.19805 Particles{Float64, 1} -sol2flux222_normal = solve(prob, alg) -sol2flux222_normal.estimated_ode_params[1] #6.05432 Particles{Float64, 1},6.0768 Particles{Float64, 1} | 6.22948 Particles{Float64, 1} -sol2flux333_normal = solve(prob, alg) -sol2flux333_normal.estimated_ode_params[1] #6.08856 Particles{Float64, 1},5.94819 Particles{Float64, 1} | 6.2551 Particles{Float64, 1} - -# LOTKA VOLTERRA CASE -function lotka_volterra(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end - -# initial-value problem. -u01 = [1.0, 1.0] -p1 = [1.5, 1.0, 3.0, 1.0] -tspan1 = (0.0, 6.0) -prob1 = ODEProblem(lotka_volterra, u01, tspan1, p1) - -# chainlux = Lux.Chain(Lux.Dense(1, 7, Lux.tanh), Lux.Dense(7, 7, Lux.tanh), Lux.Dense(7, 2)) -chainflux1 = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), Flux.Dense(8, 2)) - -#testing timepoints must match keyword arg `saveat`` timepoints of solve() call -t1 = collect(Float64, prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]) - -# -------------------------------------------------------------------------- -# original paper implementaion lotka volterra -# 31 points -solution1 = solve(prob1, Tsit5(); saveat = 0.1) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] .+ 0.3 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] .+ 0.3 .* u1[2, :] .* randn(length(u1[2, :])) -dataset2_1 = [x1, y1, time1] -plot(dataset2_1[end], dataset2_1[1]) -plot!(dataset2_1[end], dataset2_1[2]) -plot!(time1, u1[1, :]) -plot!(time1, u1[2, :]) - -alg1 = NeuralPDE.BNNODE(chainflux1, - dataset = dataset2_1, - draw_samples = 1000, - physdt = 1 / 20.0, - l2std = [ - 0.2, - 0.2, - ], - phystd = [ - 0.5, - 0.5, - ], - priorsNNw = (0.0, - 10.0), - param = [ - Normal(4, - 3), - Normal(-2, - 4), - Normal(0, - 5), - Normal(2.5, - 2)], - n_leapfrog = 30, progress = true) - -# original paper (pure data 0 1) -sol1flux1_lotka = solve(prob1, alg1) -sol1flux1_lotka.estimated_ode_params -# pure data method 1 1 -sol2flux1_lotka = solve(prob1, alg1) -sol2flux1_lotka.estimated_ode_params -# pure data method 1 0 -sol3flux1_lotka = solve(prob1, alg1) -sol3flux1_lotka.estimated_ode_params -# deri collocation -sol4flux1_lotka = solve(prob1, alg1) -sol4flux1_lotka.estimated_ode_params -# collocation -sol5flux1_lotka = solve(prob1, alg1) -sol5flux1_lotka.estimated_ode_params -# collocation + L2Data loss(at 9,0.5 1,2 gives same) -sol6flux1_lotka = solve(prob1, alg1) -sol6flux1_lotka.estimated_ode_params - -sol7flux1_lotka = solve(prob1, alg1) -sol7flux1_lotka.estimated_ode_params - -using Plots, StatsPlots -plot(dataset2_1[3], u1[1, :]) -plot!(dataset2_1[3], u1[2, :]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol5flux1_normal.ensemblesol[2]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), - sol1flux1_normal.ensemblesol[1], - legend = :outerbottomleft) -sol1flux2_normal = solve(prob1, alg1) -sol1flux2_normal.estimated_ode_params #| -sol1flux3_normal = solve(prob1, alg1) -sol1flux3_normal.estimated_ode_params #| -sol1flux4_normal = solve(prob1, alg1) -sol1flux4_normal.estimated_ode_params - -plotly() -plot!(title = "yuh") -plot!(dataset2_1[3], dataset2_1[1]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux1_normal.ensemblesol[1]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux2_normal.ensemblesol[1]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux3_normal.ensemblesol[2]) -plot!(collect(prob1.tspan[1]:(1 / 50.0):prob1.tspan[2]), sol1flux4_normal.ensemblesol[1]) -plot(time1, u1[1, :]) -plot!(time1, u1[2, :]) - -ars = chainflux1(dataset2_1[end]') -plot(ars[1, :]) -plot!(ars[2, :]) - -function calculate_derivatives(dataset) - u = dataset[1] - u1 = dataset[2] - t = dataset[end] - # control points - n = Int(floor(length(t) / 10)) - # spline for datasetvalues(solution) - # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) - interp = CubicSpline(u, t) - interp1 = CubicSpline(u1, t) - # derrivatives interpolation - dx = t[2] - t[1] - time = collect(t[1]:dx:t[end]) - smoothu = [interp(i) for i in time] - smoothu1 = [interp1(i) for i in time] - # derivative of the spline (must match function derivative) - û = tvdiff(smoothu, 20, 0.5, dx = dx, ε = 1) - û1 = tvdiff(smoothu1, 20, 0.5, dx = dx, ε = 1) - # tvdiff(smoothu, 100, 0.035, dx = dx, ε = 1) - # FDM - # û1 = diff(u) / dx - # dataset[1] and smoothu are almost equal(rounding errors) - return û, û1 - # return 1 -end - -ar = calculate_derivatives(dataset2_1) -plot(ar[1]) -plot!(ar[2]) - -# 61 points -solution1 = solve(prob1, Tsit5(); saveat = 0.1) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -dataset2_2 = [x1, y1, time1] - -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_2, - draw_samples = 1000, - l2std = [ - 0.1, - 0.1, - ], - phystd = [ - 0.1, - 0.1, - ], - priorsNNw = (0.0, - 5.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux11_normal = solve(prob1, alg1) -sol1flux11_normal.estimated_ode_params #| -sol1flux22_normal = solve(prob1, alg1) -sol1flux22_normal.estimated_ode_params #| -sol1flux33_normal = solve(prob1, alg1) -sol1flux33_normal.estimated_ode_params #| - -# 121 points -solution1 = solve(prob1, Tsit5(); saveat = 0.05) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -dataset2_3 = [x1, y1, time1] - -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_3, - draw_samples = 1000, - l2std = [ - 0.1, - 0.1, - ], - phystd = [ - 0.1, - 0.1, - ], - priorsNNw = (0.0, - 5.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux111_normal = solve(prob1, alg1) -sol1flux111_normal.estimated_ode_params #| -sol1flux222_normal = solve(prob1, alg1) -sol1flux222_normal.estimated_ode_params #| -sol1flux333_normal = solve(prob1, alg1) -sol1flux333_normal.estimated_ode_params #| - -# -------------------------------------------------------------------- - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# Sampling 100%|███████████████████████████████| Time: 0:02:30 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# Sampling 100%|███████████████████████████████| Time: 0:01:54 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# Sampling 100%|███████████████████████████████| Time: 0:01:59 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# Sampling 100%|███████████████████████████████| Time: 0:02:44 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# Sampling 100%|███████████████████████████████| Time: 0:02:41 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# Sampling 100%|███████████████████████████████| Time: 0:02:41 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# Sampling 100%|███████████████████████████████| Time: 0:03:52 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# Sampling 100%|███████████████████████████████| Time: 0:03:49 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# Sampling 100%|███████████████████████████████| Time: 0:03:50 - -# # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> -# physics Logpdf is : -6.659143464386241e7 -# prior Logpdf is : -150.30074579848434 -# L2lossData Logpdf is : -6.03075717462954e6 -# Sampling 100%|███████████████████████████████| Time: 0:04:54 - -# physics Logpdf is : -8.70012053004202e8 -# prior Logpdf is : -150.3750892952511 -# L2lossData Logpdf is : -6.967914805207133e6 -# Sampling 100%|███████████████████████████████| Time: 0:05:09 - -# physics Logpdf is : -5.417241281343099e7 -# prior Logpdf is : -150.52079555737976 -# L2lossData Logpdf is : -4.195953436792884e6 -# Sampling 100%|███████████████████████████████| Time: 0:05:01 - -# physics Logpdf is : -4.579552981943833e8 -# prior Logpdf is : -150.30491731974283 -# L2lossData Logpdf is : -8.595475827260146e6 -# Sampling 100%|███████████████████████████████| Time: 0:06:08 - -# physics Logpdf is : -1.989281834955769e7 -# prior Logpdf is : -150.16009042727543 -# L2lossData Logpdf is : -1.121270659669029e7 -# Sampling 100%|███████████████████████████████| Time: 0:05:38 - -# physics Logpdf is : -8.683829147264534e8 -# prior Logpdf is : -150.37824872259102 -# L2lossData Logpdf is : -1.0887662888035845e7 -# Sampling 100%|███████████████████████████████| Time: 0:05:50 - -# physics Logpdf is : -3.1944760610332566e8 -# prior Logpdf is : -150.33610348737565 -# L2lossData Logpdf is : -1.215458786744478e7 -# Sampling 100%|███████████████████████████████| Time: 0:10:50 - -# physics Logpdf is : -3.2884572300341567e6 -# prior Logpdf is : -150.21002268156343 -# L2lossData Logpdf is : -1.102536731511176e7 -# Sampling 100%|███████████████████████████████| Time: 0:09:53 - -# physics Logpdf is : -5.31293521002414e8 -# prior Logpdf is : -150.20948536040126 -# L2lossData Logpdf is : -1.818717239584132e7 -# Sampling 100%|███████████████████████████████| Time: 0:08:53 - -# ---------------------------------------------------------- -# Full likelihood no l2 only new L22(NN gradients) -# 25 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_new = solve(prob, alg) -sol2flux1_new.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 6.21662 Particles{Float64, 1} -sol2flux2_new = solve(prob, alg) -sol2flux2_new.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 7.14238 Particles{Float64, 1} -sol2flux3_new = solve(prob, alg) -sol2flux3_new.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 6.79159 Particles{Float64, 1} - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_new = solve(prob, alg) -sol2flux11_new.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 5.33467 Particles{Float64, 1} -sol2flux22_new = solve(prob, alg) -sol2flux22_new.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.52419 Particles{Float64, 1} -sol2flux33_new = solve(prob, alg) -sol2flux33_new.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 5.36921 Particles{Float64, 1} - -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_new = solve(prob, alg) -sol2flux111_new.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.45333 Particles{Float64, 1} -sol2flux222_new = solve(prob, alg) -sol2flux222_new.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 4.64417 Particles{Float64, 1} -sol2flux333_new = solve(prob, alg) -sol2flux333_new.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | 5.88037 Particles{Float64, 1} -# --------------------------------------------------------------------------- - -# ---------------------------------------------------------- -# Full likelihood l2 + new L22(NN gradients) -# 25 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_new_all = solve(prob, alg) -sol2flux1_new_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 6.4358 Particles{Float64, 1} -sol2flux2_new_all = solve(prob, alg) -sol2flux2_new_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 6.52449 Particles{Float64, 1} -sol2flux3_new_all = solve(prob, alg) -sol2flux3_new_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 6.34188 Particles{Float64, 1} - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_new_all = solve(prob, alg) -sol2flux11_new_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 6.37889 Particles{Float64, 1} -sol2flux22_new_all = solve(prob, alg) -sol2flux22_new_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.34747 Particles{Float64, 1} -sol2flux33_new_all = solve(prob, alg) -sol2flux33_new_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 6.39699 Particles{Float64, 1} - -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_new_all = solve(prob, alg) -sol2flux111_new_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.24327 Particles{Float64, 1} -sol2flux222_new_all = solve(prob, alg) -sol2flux222_new_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 6.23928 Particles{Float64, 1} -sol2flux333_new_all = solve(prob, alg) -sol2flux333_new_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | 6.2145 Particles{Float64, 1} - -# --------------------------------------------------------------------------- -# Full likelihood l2 + new L22(dataset gradients) lotka volterra -# 36 points -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_1, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux1_new_all = solve(prob1, alg1) -sol1flux1_new_all.estimated_ode_params[1] #| -sol1flux2_new_all = solve(prob1, alg1) -sol1flux2_new_all.estimated_ode_params[1] #| -sol1flux3_new_all = solve(prob1, alg1) -sol1flux3_new_all.estimated_ode_params[1] #| - -# 61 points -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_2, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux11_new_all = solve(prob1, alg1) -sol1flux11_new_all.estimated_ode_params[1] #| -sol1flux22_new_all = solve(prob1, alg1) -sol1flux22_new_all.estimated_ode_params[1] #| -sol1flux33_new_all = solve(prob1, alg1) -sol1flux33_new_all.estimated_ode_params[1] #| - -# 121 points -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_3, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux111_new_all = solve(prob1, alg1) -sol1flux111_new_all.estimated_ode_params[1] #| -sol1flux222_new_all = solve(prob1, alg1) -sol1flux222_new_all.estimated_ode_params[1] #| -sol1flux333_new_all = solve(prob1, alg1) -sol1flux333_new_all.estimated_ode_params[1] #| -# -------------------------------------------------------------------- - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# L2loss2 Logpdf is : -757.9047847584478 -# Sampling 100%|███████████████████████████████| Time: 0:02:32 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# L2loss2 Logpdf is : -757.9047847584478 -# Sampling 100%|███████████████████████████████| Time: 0:02:19 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -640.4155412187399 -# L2loss2 Logpdf is : -757.9047847584478 -# Sampling 100%|███████████████████████████████| Time: 0:02:31 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# L2loss2 Logpdf is : -1517.3653615845183 -# Sampling 100%|███████████████████████████████| Time: 0:03:45 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# L2loss2 Logpdf is : -1517.3653615845183 -# Sampling 100%|███████████████████████████████| Time: 0:03:20 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1198.9147562830894 -# L2loss2 Logpdf is : -1517.3653615845183 -# Sampling 100%|███████████████████████████████| Time: 0:03:20 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# L2loss2 Logpdf is : -3037.8868319811254 -# Sampling 100%|███████████████████████████████| Time: 0:04:57 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# L2loss2 Logpdf is : -3037.8868319811254 -# Sampling 100%|███████████████████████████████| Time: 0:05:26 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -2473.741390504424 -# L2loss2 Logpdf is : -3037.8868319811254 -# Sampling 100%|███████████████████████████████| Time: 0:05:01 - -# ---------------------------------------------------------- -# Full likelihood l2 + new L22(dataset gradients) -# 25 points -# 1*,2*, -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_newdata_all = solve(prob, alg) -sol2flux1_newdata_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | 5.73072 Particles{Float64, 1} -sol2flux2_newdata_all = solve(prob, alg) -sol2flux2_newdata_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | 5.71597 Particles{Float64, 1} -sol2flux3_newdata_all = solve(prob, alg) -sol2flux3_newdata_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | 5.7313 Particles{Float64, 1} - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_newdata_all = solve(prob, alg) -sol2flux11_newdata_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | 6.07153 Particles{Float64, 1} -sol2flux22_newdata_all = solve(prob, alg) -sol2flux22_newdata_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | 6.06623 Particles{Float64, 1} -sol2flux33_newdata_all = solve(prob, alg) -sol2flux33_newdata_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | 6.12748 Particles{Float64, 1} - -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_newdata_all = solve(prob, alg) -sol2flux111_newdata_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | 6.26222 Particles{Float64, 1} -sol2flux222_newdata_all = solve(prob, alg) -sol2flux222_newdata_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | 5.86494 Particles{Float64, 1} -sol2flux333_newdata_all = solve(prob, alg) -sol2flux333_newdata_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | - -# --------------------------------------------------------------------------- - -# LOTKA VOLTERRA CASE -using Plots, StatsPlots -function lotka_volterra(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end - -# initial-value problem. -u01 = [1.0, 1.0] -p1 = [1.5, 1.0, 3.0, 1.0] -tspan1 = (0.0, 6.0) -prob1 = ODEProblem(lotka_volterra, u01, tspan1, p1) - -chainlux = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), Lux.Dense(6, 2)) - -#testing timepoints must match keyword arg `saveat`` timepoints of solve() call -t1 = collect(Float64, prob.tspan[1]:(1 / 50.0):prob.tspan[2]) - -# -------------------------------------------------------------------------- -# original paper implementaion -# 25 points -solution1 = solve(prob1, Tsit5(); saveat = 0.2) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -dataset2_1 = [x1, y1, time1] - -plot(time1, u1[1, :]) -plot!(time1, u1[2, :]) -scatter!(dataset2_1[3], dataset2_1[1]) -scatter!(dataset2_1[3], dataset2_1[2]) - -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_1, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux1_normal = solve(prob1, alg1) -sol1flux1_normal.estimated_ode_params[1] #| -sol1flux2_normal = solve(prob1, alg1) -sol1flux2_normal.estimated_ode_params[1] #| -sol1flux3_normal = solve(prob1, alg1) -sol1flux3_normal.estimated_ode_params[1] #| - -# 50 points -solution1 = solve(prob1, Tsit5(); saveat = 0.05) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -dataset2_2 = [x1, y1, time1] - -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_2, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux11_normal = solve(prob1, alg1) -sol1flux11_normal.estimated_ode_params[1] #| -sol1flux22_normal = solve(prob1, alg1) -sol1flux22_normal.estimated_ode_params[1] #| -sol1flux33_normal = solve(prob1, alg1) -sol1flux33_normal.estimated_ode_params[1] #| - -# 100 points -solution = solve(prob1, Tsit5(); saveat = 0.05) -time1 = solution1.t -physsol1_1 = solution1.u -u1 = hcat(solution1.u...) -x1 = u1[1, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -y1 = u1[2, :] + 0.4 .* u1[1, :] .* randn(length(u1[1, :])) -dataset2_3 = [x1, y1, time1] - -alg1 = NeuralPDE.BNNODE(chainlux, - dataset = dataset2_3, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol1flux111_normal = solve(prob1, alg1) -sol1flux111_normal.estimated_ode_params[1] #| -sol1flux222_normal = solve(prob1, alg1) -sol1flux222_normal.estimated_ode_params[1] #| -sol1flux333_normal = solve(prob1, alg1) -sol1flux333_normal.estimated_ode_params[1] #| - -# -------------------------------------------------------------------- - -# ---------------------------------------------------------- -# Full likelihood no l2 only new L22(NN gradients) -# 25 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_new = solve(prob, alg) -sol2flux1_new.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | -sol2flux2_new = solve(prob, alg) -sol2flux2_new.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | -sol2flux3_new = solve(prob, alg) -sol2flux3_new.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_new = solve(prob, alg) -sol2flux11_new.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | -sol2flux22_new = solve(prob, alg) -sol2flux22_new.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | -sol2flux33_new = solve(prob, alg) -sol2flux33_new.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | - -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_new = solve(prob, alg) -sol2flux111_new.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | -sol2flux222_new = solve(prob, alg) -sol2flux222_new.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | -sol2flux333_new = solve(prob, alg) -sol2flux333_new.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | -# --------------------------------------------------------------------------- - -# ---------------------------------------------------------- -# Full likelihood l2 + new L22(NN gradients) -# 25 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux1_new_all = solve(prob, alg) -sol2flux1_new_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | -sol2flux2_new_all = solve(prob, alg) -sol2flux2_new_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | -sol2flux3_new_all = solve(prob, alg) -sol2flux3_new_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux11_new_all = solve(prob, alg) -sol2flux11_new_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | -sol2flux22_new_all = solve(prob, alg) -sol2flux22_new_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | -sol2flux33_new_all = solve(prob, alg) -sol2flux33_new_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol2flux111_new_all = solve(prob, alg) -sol2flux111_new_all.estimated_ode_params[1] #6.94385 Particles{Float64, 1},5.87832 Particles{Float64, 1} | -sol2flux222_new_all = solve(prob, alg) -sol2flux222_new_all.estimated_ode_params[1] #5.888 Particles{Float64, 1},5.86901 Particles{Float64, 1} | -sol2flux333_new_all = solve(prob, alg) -sol2flux333_new_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | - -# --------------------------------------------------------------------------- - -# ---------------------------------------------------------- -# Full likelihood l2 + new L22(dataset gradients) -# 25 points -# *1,*2 vs *2.5 -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset1, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol1flux1_newdata_all = solve(prob, alg) -sol1flux1_newdata_all.estimated_ode_params[1] #5.35705 Particles{Float64, 1},5.91809 Particles{Float64, 1} | -sol1flux2_newdata_all = solve(prob, alg) -sol1flux2_newdata_all.estimated_ode_params[1] #6.73629 Particles{Float64, 1},5.966 Particles{Float64, 1} | -sol1flux3_newdata_all = solve(prob, alg) -sol1flux3_newdata_all.estimated_ode_params[1] #4.64324 Particles{Float64, 1},5.9559 Particles{Float64, 1} | - -# 50 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset2, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol1flux11_newdata_all = solve(prob, alg) -sol1flux11_newdata_all.estimated_ode_params[1] #6.43659 Particles{Float64, 1},6.03723 Particles{Float64, 1} | -sol1flux22_newdata_all = solve(prob, alg) -sol1flux22_newdata_all.estimated_ode_params[1] # 6.4389 Particles{Float64, 1},6.01308 Particles{Float64, 1} | -sol1flux33_newdata_all = solve(prob, alg) -sol1flux33_newdata_all.estimated_ode_params[1] # 7.10082 Particles{Float64, 1}, 6.03989 Particles{Float64, 1} | - -# 100 points -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset3, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -sol1flux111_newdata_all = solve(prob, alg) -sol1flux111_newdata_all.estimated_ode_params[1] #| -sol1flux222_newdata_all = solve(prob, alg) -sol1flux222_newdata_all.estimated_ode_params[1] #| -sol1flux333_newdata_all = solve(prob, alg) -sol1flux333_newdata_all.estimated_ode_params[1] #6.96835 Particles{Float64, 1},5.86708 Particles{Float64, 1} | - -# ------------------------------------------------------------------------------------------------------------------------------ - -# sol2flux111.estimated_ode_params[1] -# # mine *5 -# 7.03386Particles{Float64, 1} -# # normal -# 6.38951Particles{Float64, 1} -# 6.67657Particles{Float64, 1} -# # mine *10 -# 7.53672Particles{Float64, 1} -# # mine *2 -# 6.29005Particles{Float64, 1} -# 6.29844Particles{Float64, 1} - -# # new mine *2 -# 6.39008Particles{Float64, 1} -# 6.22071Particles{Float64, 1} -# 6.15611Particles{Float64, 1} - -# # new mine *2 tvdiff(smoothu, 20, 0.035, dx = dx, ε = 1e-2) -# 6.25549Particles{Float64, 1} -# ---------------------------------------------------------- - -# --------------------------------------------------- - -function calculate_derivatives1(dataset) - x̂, time = dataset - num_points = length(x̂) - # Initialize an array to store the derivative values. - derivatives = similar(x̂) - - for i in 2:(num_points - 1) - # Calculate the first-order derivative using central differences. - Δt_forward = time[i + 1] - time[i] - Δt_backward = time[i] - time[i - 1] - - derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) - - derivatives[i] = derivative - end - - # Derivatives at the endpoints can be calculated using forward or backward differences. - derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) - derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) - return derivatives -end - -function calculate_derivatives2(dataset) - u = dataset[1] - t = dataset[2] - # control points - n = Int(floor(length(t) / 10)) - # spline for datasetvalues(solution) - # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) - interp = CubicSpline(u, t) - # derrivatives interpolation - dx = t[2] - t[1] - time = collect(t[1]:dx:t[end]) - smoothu = [interp(i) for i in time] - # derivative of the spline (must match function derivative) - û = tvdiff(smoothu, 20, 0.03, dx = dx, ε = 1) - # tvdiff(smoothu, 100, 0.1, dx = dx) - # - # - # FDM - û1 = diff(u) / dx - # dataset[1] and smoothu are almost equal(rounding errors) - return û, time, smoothu, û1 -end - -# need to do this for all datasets -c = [linear(prob.u0, p, t) for t in dataset3[2]] #ideal case -b = calculate_derivatives1(dataset2) #central diffs -# a = calculate_derivatives2(dataset) #tvdiff(smoothu, 100, 0.1, dx = dx) -d = calculate_derivatives2(dataset1) #tvdiff(smoothu, 20, 0.035, dx = dx, ε = 1e-2) -d = calculate_derivatives2(dataset2) -d = calculate_derivatives2(dataset3) -mean(abs2.(c .- b)) -mean(abs2.(c .- d[1])) -loss(model, x, y) = mean(abs2.(model(x) .- y)); -scatter!(prob.u0 .+ (prob.tspan[2] .- dataset3[2]) .* chainflux1(dataset3[2]')') -loss(chainflux1, dataset3[2]', dataset3[1]') -# mean(abs2.(c[1:24] .- a[4])) -plot(c, label = "ideal deriv") -plot!(b, label = "Centraldiff deriv") -# plot!(a[1], label = "tvdiff(0.1,def) derivatives") -plot!(d[1], label = "tvdiff(0.035,20) derivatives") -plotly() - -# GridTraining , NoiseRobustDiff dataset[2][2]-dataset[2][1] l2std -# 25 points -ta = range(tspan[1], tspan[2], length = 25) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -time1 = collect(tspan[1]:(1 / 50.0):tspan[2]) -physsol = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] -plot(physsol, label = "solution") - -# plots from 32(deriv) -# for d -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 2000, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux1 = solve(prob, alg) -n2_sol2flux1.estimated_ode_params[1] -# with extra likelihood -# 10.2011Particles{Float64, 1} - -# without extra likelihood -# 6.25791Particles{Float64, 1} -# 6.29539Particles{Float64, 1} - -plot!(n2_sol2flux1.ensemblesol[1], label = "tvdiff(0.035,1) derivpar") -plot(dataset[1]) -plot!(physsol1) -# for a -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 2000, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux2 = solve(prob, alg) -n2_sol2flux2.estimated_ode_params[1] -# with extra likelihood -# 8.73602Particles{Float64, 1} -# without extra likelihood - -plot!(n2_sol2flux2.ensemblesol[1], - label = "tvdiff(0.1,def) derivatives", - legend = :outerbottomleft) - -# for b -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 2000, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux3 = solve(prob, alg) -n2_sol2flux3.estimated_ode_params[1] -plot!(n2_sol2flux3.ensemblesol[1], label = "Centraldiff deriv") - -# for c -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 2000, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux4 = solve(prob, alg) -n2_sol2flux4.estimated_ode_params[1] -plot!(n2_sol2flux4.ensemblesol[1], label = "ideal deriv") - -# 50 points - -ta = range(tspan[1], tspan[2], length = 50) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux11 = solve(prob, alg) -n2_sol2flux11.estimated_ode_params[1] - -# 5.90049Particles{Float64, 1} -# 100 points -ta = range(tspan[1], tspan[2], length = 100) -u = [linear_analytic(u0, p, ti) for ti in ta] -x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -time = vec(collect(Float64, ta)) -dataset = [x̂, time] -physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, - draw_samples = 1500, physdt = 1 / 50.0f0, - priorsNNw = (0.0, 3.0), - param = [LogNormal(9, 0.5)], - Metric = DiagEuclideanMetric, - n_leapfrog = 30, progress = true) - -n2_sol2flux111 = solve(prob, alg) -n2_sol2flux111.estimated_ode_params[1] -plot!(n2_sol2flux111.ensemblesol[1]) -8.88555Particles{Float64, 1} - -# 7.15353Particles{Float64, 1} -# 6.21059 Particles{Float64, 1} -# 6.31836Particles{Float64, 1} -0.1 * p -# ---------------------------------------------------------- - -# Gives the linear interpolation value at t=3.5 - -# # Problem 1 with param esimation -# # dataset 0-1 2 percent noise -# p = 6.283185307179586 -# # partial_logdensity -# 6.3549Particles{Float64, 1} -# # full log_density -# 6.34667Particles{Float64, 1} - -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2lux.estimated_ode_params[1] - -# # dataset 0-1 20 percent noise -# # partial log_density -# 6.30244Particles{Float64, 1} -# # full log_density -# 6.24637Particles{Float64, 1} - -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # dataset 0-2 20percent noise -# # partial log_density -# 6.24948Particles{Float64, 1} -# # full log_density -# 6.26095Particles{Float64, 1} - -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# linear_analytic = (u0, p, t) -> u0 + sin(p * t) / (p) -# linear = (u, p, t) -> cos(p * t) -# tspan = (0.0, 2.0) - -# # dataset 0-1 2 percent noise -# p = 6.283185307179586 -# # partial_logdensity -# 6.3549Particles{Float64, 1} -# # full log_density -# 6.34667Particles{Float64, 1} - -# # dataset 0-1 20 percent noise -# # partial log_density -# 6.30244Particles{Float64, 1} -# # full log_density -# 6.24637Particles{Float64, 1} - -# # dataset 0-2 20percent noise -# # partial log_density -# 6.24948Particles{Float64, 1} -# # full log_density -# 6.26095Particles{Float64, 1} - -# # dataset 0-2 20percent noise 50 points(above all are 100 points) -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # partial log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# # i kinda win on 25 points again -# # dataset 0-2 20percent noise 25 points -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # partial log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # i win with 25 points -# # dataset 0-1 20percent noise 25 points -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# # new -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # partial log_density -# sol2flux.estimated_ode_params[1] -# # New -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # (9,2.5)(above are (9,0.5)) -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# # just prev was repeat(just change) -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # partial log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # i lose on 0-1,50 points -# # dataset 0-1 20percent noise 50 points -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] -# # partial log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # (9,2.5) (above are (9,0.5)) -# # FuLL log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # partial log_density -# sol2flux.estimated_ode_params[1] -# sol2flux.estimated_ode_params[1] - -# # ---------------------------------------------------------- -# # Problem 1 with param estimation -# # physdt=1/20, Full likelihood new 0.5*l2std -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n05_sol2flux1 = solve(prob, alg) -# n05_sol2flux1.estimated_ode_params[1] #6.90953 Particles{Float64, 1} -# n05_sol2flux2 = solve(prob, alg) -# n05_sol2flux2.estimated_ode_params[1] #6.82374 Particles{Float64, 1} -# n05_sol2flux3 = solve(prob, alg) -# n05_sol2flux3.estimated_ode_params[1] #6.84465 Particles{Float64, 1} - -# using Plots, StatsPlots -# plot(n05_sol2flux3.ensemblesol[1]) -# plot!(physsol1) -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n05_sol2flux11 = solve(prob, alg) -# n05_sol2flux11.estimated_ode_params[1] #7.0262 Particles{Float64, 1} -# n05_sol2flux22 = solve(prob, alg) -# n05_sol2flux22.estimated_ode_params[1] #5.56438 Particles{Float64, 1} -# n05_sol2flux33 = solve(prob, alg) -# n05_sol2flux33.estimated_ode_params[1] #7.27189 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n05_sol2flux111 = solve(prob, alg) -# n05_sol2flux111.estimated_ode_params[1] #6.90549 Particles{Float64, 1} -# n05_sol2flux222 = solve(prob, alg) -# n05_sol2flux222.estimated_ode_params[1] #5.42436 Particles{Float64, 1} -# n05_sol2flux333 = solve(prob, alg) -# n05_sol2flux333.estimated_ode_params[1] #6.05832 Particles{Float64, 1} - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood new 2*l2std -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2_sol2flux1 = solve(prob, alg) -# n2_sol2flux1.estimated_ode_params[1]#6.9087 Particles{Float64, 1} -# n2_sol2flux2 = solve(prob, alg) -# n2_sol2flux2.estimated_ode_params[1]#6.86507 Particles{Float64, 1} -# n2_sol2flux3 = solve(prob, alg) -# n2_sol2flux3.estimated_ode_params[1]#6.59206 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2_sol2flux11 = solve(prob, alg) -# n2_sol2flux11.estimated_ode_params[1]#7.3715 Particles{Float64, 1} -# n2_sol2flux22 = solve(prob, alg) -# n2_sol2flux22.estimated_ode_params[1]#9.84477 Particles{Float64, 1} -# n2_sol2flux33 = solve(prob, alg) -# n2_sol2flux33.estimated_ode_params[1]#6.87107 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2_sol2flux111 = solve(prob, alg) -# n2_sol2flux111.estimated_ode_params[1]#6.60739 Particles{Float64, 1} -# n2_sol2flux222 = solve(prob, alg) -# n2_sol2flux222.estimated_ode_params[1]#7.05923 Particles{Float64, 1} -# n2_sol2flux333 = solve(prob, alg) -# n2_sol2flux333.estimated_ode_params[1]#6.5017 Particles{Float64, 1} - -# # ---------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood new all 2*l2std -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2all5sol2flux1 = solve(prob, alg) -# n2all5sol2flux1.estimated_ode_params[1]#11.3659 Particles{Float64, 1} -# n2all5sol2flux2 = solve(prob, alg) -# n2all5sol2flux2.estimated_ode_params[1]#6.65634 Particles{Float64, 1} -# n2all5sol2flux3 = solve(prob, alg) -# n2all5sol2flux3.estimated_ode_params[1]#6.61905 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2all5sol2flux11 = solve(prob, alg) -# n2all5sol2flux11.estimated_ode_params[1]#6.27555 Particles{Float64, 1} -# n2all5sol2flux22 = solve(prob, alg) -# n2all5sol2flux22.estimated_ode_params[1]#6.24352 Particles{Float64, 1} -# n2all5sol2flux33 = solve(prob, alg) -# n2all5sol2flux33.estimated_ode_params[1]#6.33723 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n2all5sol2flux111 = solve(prob, alg) -# n2all5sol2flux111.estimated_ode_params[1] #5.95535 Particles{Float64, 1} -# n2all5sol2flux222 = solve(prob, alg) -# n2all5sol2flux222.estimated_ode_params[1] #5.98301 Particles{Float64, 1} -# n2all5sol2flux333 = solve(prob, alg) -# n2all5sol2flux333.estimated_ode_params[1] #5.9081 Particles{Float64, 1} - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood new all (l2+l22) -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nall5sol2flux1 = solve(prob, alg) -# nall5sol2flux1.estimated_ode_params[1]#6.54705 Particles{Float64, 1} -# nall5sol2flux2 = solve(prob, alg) -# nall5sol2flux2.estimated_ode_params[1]#6.6967 Particles{Float64, 1} -# nall5sol2flux3 = solve(prob, alg) -# nall5sol2flux3.estimated_ode_params[1]#6.47173 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nall5sol2flux11 = solve(prob, alg) -# nall5sol2flux11.estimated_ode_params[1]#6.2113 Particles{Float64, 1} -# nall5sol2flux22 = solve(prob, alg) -# nall5sol2flux22.estimated_ode_params[1]#6.10675 Particles{Float64, 1} -# nall5sol2flux33 = solve(prob, alg) -# nall5sol2flux33.estimated_ode_params[1]#6.11541 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nall5sol2flux111 = solve(prob, alg) -# nall5sol2flux111.estimated_ode_params[1]#6.35224 Particles{Float64, 1} -# nall5sol2flux222 = solve(prob, alg) -# nall5sol2flux222.estimated_ode_params[1]#6.40542 Particles{Float64, 1} -# nall5sol2flux333 = solve(prob, alg) -# nall5sol2flux333.estimated_ode_params[1]#6.44206 Particles{Float64, 1} - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood new 5* (new only l22 mod) -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n5sol2flux1 = solve(prob, alg) -# n5sol2flux1.estimated_ode_params[1]#7.05077 Particles{Float64, 1} -# n5sol2flux2 = solve(prob, alg) -# n5sol2flux2.estimated_ode_params[1]#7.07303 Particles{Float64, 1} -# n5sol2flux3 = solve(prob, alg) -# n5sol2flux3.estimated_ode_params[1]#5.10622 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n5sol2flux11 = solve(prob, alg) -# n5sol2flux11.estimated_ode_params[1]#7.39852 Particles{Float64, 1} -# n5sol2flux22 = solve(prob, alg) -# n5sol2flux22.estimated_ode_params[1]#7.30319 Particles{Float64, 1} -# n5sol2flux33 = solve(prob, alg) -# n5sol2flux33.estimated_ode_params[1]#6.73722 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# n5sol2flux111 = solve(prob, alg) -# n5sol2flux111.estimated_ode_params[1]#7.15996 Particles{Float64, 1} -# n5sol2flux222 = solve(prob, alg) -# n5sol2flux222.estimated_ode_params[1]#7.02949 Particles{Float64, 1} -# n5sol2flux333 = solve(prob, alg) -# n5sol2flux333.estimated_ode_params[1]#6.9393 Particles{Float64, 1} - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood new -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nsol2flux1 = solve(prob, alg) -# nsol2flux1.estimated_ode_params[1] #5.82707 Particles{Float64, 1} -# nsol2flux2 = solve(prob, alg) -# nsol2flux2.estimated_ode_params[1] #4.81534 Particles{Float64, 1} -# nsol2flux3 = solve(prob, alg) -# nsol2flux3.estimated_ode_params[1] #5.52965 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nsol2flux11 = solve(prob, alg) -# nsol2flux11.estimated_ode_params[1] #7.04027 Particles{Float64, 1} -# nsol2flux22 = solve(prob, alg) -# nsol2flux22.estimated_ode_params[1] #7.17588 Particles{Float64, 1} -# nsol2flux33 = solve(prob, alg) -# nsol2flux33.estimated_ode_params[1] #6.94495 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# nsol2flux111 = solve(prob, alg) -# nsol2flux111.estimated_ode_params[1] #6.06608 Particles{Float64, 1} -# nsol2flux222 = solve(prob, alg) -# nsol2flux222.estimated_ode_params[1] #6.84726 Particles{Float64, 1} -# nsol2flux333 = solve(prob, alg) -# nsol2flux333.estimated_ode_params[1] #6.83463 Particles{Float64, 1} - -# # ---------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux1 = solve(prob, alg) -# sol2flux1.estimated_ode_params[1] #6.71397 Particles{Float64, 1} 6.37604 Particles{Float64, 1} -# sol2flux2 = solve(prob, alg) -# sol2flux2.estimated_ode_params[1] #6.73509 Particles{Float64, 1} 6.21692 Particles{Float64, 1} -# sol2flux3 = solve(prob, alg) -# sol2flux3.estimated_ode_params[1] #6.65453 Particles{Float64, 1} 6.23153 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux11 = solve(prob, alg) -# sol2flux11.estimated_ode_params[1] #6.23443 Particles{Float64, 1} 6.30635 Particles{Float64, 1} -# sol2flux22 = solve(prob, alg) -# sol2flux22.estimated_ode_params[1] #6.18879 Particles{Float64, 1} 6.30099 Particles{Float64, 1} -# sol2flux33 = solve(prob, alg) -# sol2flux33.estimated_ode_params[1] #6.22773 Particles{Float64, 1} 6.30671 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux111 = solve(prob, alg) -# sol2flux111.estimated_ode_params[1] #6.15832 Particles{Float64, 1} 6.35453 Particles{Float64, 1} -# sol2flux222 = solve(prob, alg) -# sol2flux222.estimated_ode_params[1] #6.16968 Particles{Float64, 1}6.31125 Particles{Float64, 1} -# sol2flux333 = solve(prob, alg) -# sol2flux333.estimated_ode_params[1] #6.12466 Particles{Float64, 1} 6.26514 Particles{Float64, 1} - -# # ---------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, partial likelihood -# # 25 points -# ta = range(tspan[1], tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux1_p = solve(prob, alg) -# sol2flux1_p.estimated_ode_params[1] #5.74065 Particles{Float64, 1} #6.83683 Particles{Float64, 1} -# sol2flux2_p = solve(prob, alg) -# sol2flux2_p.estimated_ode_params[1] #9.82504 Particles{Float64, 1} #6.14568 Particles{Float64, 1} -# sol2flux3_p = solve(prob, alg) -# sol2flux3_p.estimated_ode_params[1] #5.75075 Particles{Float64, 1} #6.08579 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux11_p = solve(prob, alg) -# sol2flux11_p.estimated_ode_params[1] #6.19414 Particles{Float64, 1} #6.04621 Particles{Float64, 1} -# sol2flux22_p = solve(prob, alg) -# sol2flux22_p.estimated_ode_params[1] #6.15227 Particles{Float64, 1} #6.29086 Particles{Float64, 1} -# sol2flux33_p = solve(prob, alg) -# sol2flux33_p.estimated_ode_params[1] #6.19048 Particles{Float64, 1} #6.12516 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol2flux111_p = solve(prob, alg) -# sol2flux111_p.estimated_ode_params[1] #6.51608 Particles{Float64, 1}# 6.42945Particles{Float64, 1} -# sol2flux222_p = solve(prob, alg) -# sol2flux222_p.estimated_ode_params[1] #6.4875 Particles{Float64, 1} # 6.44524Particles{Float64, 1} -# sol2flux333_p = solve(prob, alg) -# sol2flux333_p.estimated_ode_params[1] #6.51679 Particles{Float64, 1}# 6.43152Particles{Float64, 1} - -# # --------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood, dataset(1.0-2.0) -# # 25 points -# ta = range(1.0, tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux1 = solve(prob, alg) -# sol1flux1.estimated_ode_params[1] #6.35164 Particles{Float64, 1} -# sol1flux2 = solve(prob, alg) -# sol1flux2.estimated_ode_params[1] #6.30919 Particles{Float64, 1} -# sol1flux3 = solve(prob, alg) -# sol1flux3.estimated_ode_params[1] #6.33554 Particles{Float64, 1} - -# # 50 points -# ta = range(1.0, tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux11 = solve(prob, alg) -# sol1flux11.estimated_ode_params[1] #6.39769 Particles{Float64, 1} -# sol1flux22 = solve(prob, alg) -# sol1flux22.estimated_ode_params[1] #6.43924 Particles{Float64, 1} -# sol1flux33 = solve(prob, alg) -# sol1flux33.estimated_ode_params[1] #6.4697 Particles{Float64, 1} - -# # 100 points -# ta = range(1.0, tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux111 = solve(prob, alg) -# sol1flux111.estimated_ode_params[1] #6.27812 Particles{Float64, 1} -# sol1flux222 = solve(prob, alg) -# sol1flux222.estimated_ode_params[1] #6.19278 Particles{Float64, 1} -# sol1flux333 = solve(prob, alg) -# sol1flux333.estimated_ode_params[1] # 9.68244Particles{Float64, 1} (first try) # 6.23969 Particles{Float64, 1}(second try) - -# # ---------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, partial likelihood, dataset(1.0-2.0) -# # 25 points -# ta = range(1.0, tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux1_p = solve(prob, alg) -# sol1flux1_p.estimated_ode_params[1]#6.36269 Particles{Float64, 1} - -# sol1flux2_p = solve(prob, alg) -# sol1flux2_p.estimated_ode_params[1]#6.34685 Particles{Float64, 1} - -# sol1flux3_p = solve(prob, alg) -# sol1flux3_p.estimated_ode_params[1]#6.31421 Particles{Float64, 1} - -# # 50 points -# ta = range(1.0, tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux11_p = solve(prob, alg) -# sol1flux11_p.estimated_ode_params[1] #6.15725 Particles{Float64, 1} - -# sol1flux22_p = solve(prob, alg) -# sol1flux22_p.estimated_ode_params[1] #6.18145 Particles{Float64, 1} - -# sol1flux33_p = solve(prob, alg) -# sol1flux33_p.estimated_ode_params[1] #6.21905 Particles{Float64, 1} - -# # 100 points -# ta = range(1.0, tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1flux111_p = solve(prob, alg) -# sol1flux111_p.estimated_ode_params[1]#6.13481 Particles{Float64, 1} - -# sol1flux222_p = solve(prob, alg) -# sol1flux222_p.estimated_ode_params[1]#9.68555 Particles{Float64, 1} - -# sol1flux333_p = solve(prob, alg) -# sol1flux333_p.estimated_ode_params[1]#6.1477 Particles{Float64, 1} - -# # ----------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, partial likelihood, dataset(1-2), again but different density -# # 12 points -# ta = range(1.0, tspan[2], length = 12) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol3flux1_p = solve(prob, alg) -# sol3flux1_p.estimated_ode_params[1]#6.50048 Particles{Float64, 1} -# sol3flux2_p = solve(prob, alg) -# sol3flux2_p.estimated_ode_params[1]#6.57597 Particles{Float64, 1} -# sol3flux3_p = solve(prob, alg) -# sol3flux3_p.estimated_ode_params[1]#6.24487 Particles{Float64, 1} - -# # 25 points -# ta = range(1.0, tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol3flux11_p = solve(prob, alg) -# sol3flux11_p.estimated_ode_params[1]#6.53093 Particles{Float64, 1} - -# sol3flux22_p = solve(prob, alg) -# sol3flux22_p.estimated_ode_params[1]#6.32744 Particles{Float64, 1} - -# sol3flux33_p = solve(prob, alg) -# sol3flux33_p.estimated_ode_params[1]#6.49175 Particles{Float64, 1} - -# # 50 points -# ta = range(1.0, tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol3flux111_p = solve(prob, alg) -# sol3flux111_p.estimated_ode_params[1]#6.4455 Particles{Float64, 1} -# sol3flux222_p = solve(prob, alg) -# sol3flux222_p.estimated_ode_params[1]#6.40736 Particles{Float64, 1} -# sol3flux333_p = solve(prob, alg) -# sol3flux333_p.estimated_ode_params[1]#6.46214 Particles{Float64, 1} - -# # --------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, partial likelihood, dataset(0-1) -# # 25 points -# ta = range(tspan[1], 1.0, length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol0flux1_p = solve(prob, alg) -# sol0flux1_p.estimated_ode_params[1]#7.12625 Particles{Float64, 1} -# sol0flux2_p = solve(prob, alg) -# sol0flux2_p.estimated_ode_params[1]#8.40948 Particles{Float64, 1} -# sol0flux3_p = solve(prob, alg) -# sol0flux3_p.estimated_ode_params[1]#7.18768 Particles{Float64, 1} - -# # 50 points -# ta = range(tspan[1], 1.0, length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol0flux11_p = solve(prob, alg) -# sol0flux11_p.estimated_ode_params[1]#6.23707 Particles{Float64, 1} -# sol0flux22_p = solve(prob, alg) -# sol0flux22_p.estimated_ode_params[1]#6.09728 Particles{Float64, 1} -# sol0flux33_p = solve(prob, alg) -# sol0flux33_p.estimated_ode_params[1]#6.12971 Particles{Float64, 1} - -# # 100 points -# ta = range(tspan[1], 1.0, length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [LogNormal(9, 0.5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol0flux111_p = solve(prob, alg) -# sol0flux111_p.estimated_ode_params[1]#5.99039 Particles{Float64, 1} -# sol0flux222_p = solve(prob, alg) -# sol0flux222_p.estimated_ode_params[1]#5.89609 Particles{Float64, 1} -# sol0flux333_p = solve(prob, alg) -# sol0flux333_p.estimated_ode_params[1]#5.91923 Particles{Float64, 1} - -# # --------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, Full likelihood, dataset(1.0-2.0), Normal(12,5) distri prior -# # 25 points -# ta = range(1.0, tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 6.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f1 = solve(prob, alg) -# sol1f1.estimated_ode_params[1] -# # 10.9818Particles{Float64, 1} -# sol1f2 = solve(prob, alg) -# sol1f2.estimated_ode_params[1] -# # sol1f3 = solve(prob, alg) -# # sol1f3.estimated_ode_params[1] - -# # 50 points -# ta = range(1.0, tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 6.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f11 = solve(prob, alg) -# sol1f11.estimated_ode_params[1] -# sol1f22 = solve(prob, alg) -# sol1f22.estimated_ode_params[1] -# # sol1f33 = solve(prob, alg) -# # sol1f33.estimated_ode_params[1] - -# # 100 points -# ta = range(1.0, tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 6.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f111 = solve(prob, alg) -# sol1f111.estimated_ode_params[1] -# sol1f222 = solve(prob, alg) -# sol1f222.estimated_ode_params[1] -# # sol1f333 = solve(prob, alg) -# # sol1f333.estimated_ode_params[1] - -# # ---------------------------------------------------------- - -# # ---------------------------------------------------------- -# # physdt=1/20, partial likelihood, dataset(1.0-2.0), Normal(12,5) distri prior -# # 25 points -# ta = range(1.0, tspan[2], length = 25) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f1_p = solve(prob, alg) -# sol1f1_p.estimated_ode_params[1] -# sol1f2_p = solve(prob, alg) -# sol1f2_p.estimated_ode_params[1] -# sol1f3_p = solve(prob, alg) -# sol1f3_p.estimated_ode_params[1] - -# # 50 points -# ta = range(1.0, tspan[2], length = 50) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f11_p = solve(prob, alg) -# sol1f11_p.estimated_ode_params[1] -# sol1f22_p = solve(prob, alg) -# sol1f22_p.estimated_ode_params[1] -# sol1f33_p = solve(prob, alg) -# sol1f33_p.estimated_ode_params[1] - -# # 100 points -# ta = range(1.0, tspan[2], length = 100) -# u = [linear_analytic(u0, p, ti) for ti in ta] -# x̂ = collect(Float64, Array(u) + 0.2 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂, time] -# physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - -# alg = NeuralPDE.BNNODE(chainflux1, dataset = dataset, -# draw_samples = 1500, physdt = 1 / 50.0f0, -# priorsNNw = (0.0, 3.0), -# param = [Normal(12, 5)], -# Metric = DiagEuclideanMetric, -# n_leapfrog = 30, progress = true) - -# sol1f111_p = solve(prob, alg) -# sol1f111_p.estimated_ode_params[1] -# sol1f222_p = solve(prob, alg) -# sol1f222_p.estimated_ode_params[1] -# sol1f333_p = solve(prob, alg) -# sol1f333_p.estimated_ode_params[1] - -# # ---------------------------------------------------------- - -# plot!(title = "9,2.5 50 training 2>full,1>partial") - -# p -# param1 -# # (lux chain) -# @prob mean(abs.(physsol2 .- sol3lux_pestim.ensemblesol[1])) < 8e-2 - -# # estimated parameters(lux chain) -# param1 = sol3lux_pestim.estimated_ode_params[1] -# @test abs(param1 - p) < abs(0.35 * p) - -# p -# param1 - -# # # my suggested Loss likelihood part -# # # + L2loss2(Tar, θ) -# # # My suggested extra loss function -# # function L2loss2(Tar::LogTargetDensity, θ) -# # f = Tar.prob.f - -# # # parameter estimation chosen or not -# # if Tar.extraparams > 0 -# # dataset = Tar.dataset - -# # # Timepoints to enforce Physics -# # dataset = Array(reduce(hcat, dataset)') -# # t = dataset[end, :] -# # û = dataset[1:(end - 1), :] - -# # ode_params = Tar.extraparams == 1 ? -# # θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : -# # θ[((length(θ) - Tar.extraparams) + 1):length(θ)] - -# # if length(û[:, 1]) == 1 -# # physsol = [f(û[:, i][1], -# # ode_params, -# # t[i]) -# # for i in 1:length(û[1, :])] -# # else -# # physsol = [f(û[:, i], -# # ode_params, -# # t[i]) -# # for i in 1:length(û[1, :])] -# # end -# # #form of NN output matrix output dim x n -# # deri_physsol = reduce(hcat, physsol) - -# # # OG deriv(basically gradient matching in case of an ODEFunction) -# # # in case of PDE or general ODE we would want to reduce residue of f(du,u,p,t) -# # # if length(û[:, 1]) == 1 -# # # deri_sol = [f(û[:, i][1], -# # # Tar.prob.p, -# # # t[i]) -# # # for i in 1:length(û[1, :])] -# # # else -# # # deri_sol = [f(û[:, i], -# # # Tar.prob.p, -# # # t[i]) -# # # for i in 1:length(û[1, :])] -# # # end -# # # deri_sol = reduce(hcat, deri_sol) -# # derivatives = calculate_derivatives(Tar.dataset) -# # deri_sol = reduce(hcat, derivatives) - -# # physlogprob = 0 -# # for i in 1:length(Tar.prob.u0) -# # # can add phystd[i] for u[i] -# # physlogprob += logpdf(MvNormal(deri_physsol[i, :], -# # LinearAlgebra.Diagonal(map(abs2, -# # Tar.l2std[i] .* -# # ones(length(deri_sol[i, :]))))), -# # deri_sol[i, :]) -# # end -# # return physlogprob -# # else -# # return 0 -# # end -# # end - -# # function calculate_derivatives(dataset) -# # x̂, time = dataset -# # num_points = length(x̂) - -# # # Initialize an array to store the derivative values. -# # derivatives = similar(x̂) - -# # for i in 2:(num_points - 1) -# # # Calculate the first-order derivative using central differences. -# # Δt_forward = time[i + 1] - time[i] -# # Δt_backward = time[i] - time[i - 1] - -# # derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) - -# # derivatives[i] = derivative -# # end - -# # # Derivatives at the endpoints can be calculated using forward or backward differences. -# # derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) -# # derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) - -# # return derivatives -# # end - -# size(dataset[1]) -# # Problem 1 with param estimation(flux,lux) -# # Normal -# # 6.20311 Particles{Float64, 1},6.21746Particles{Float64, 1} -# # better -# # 6.29093Particles{Float64, 1}, 6.27925Particles{Float64, 1} -# # Non ideal case -# # 6.14861Particles{Float64, 1}, -# sol2flux.estimated_ode_params -# sol2lux.estimated_ode_params[1] -# p -# size(sol3flux_pestim.ensemblesol[2]) -# plott = sol3flux_pestim.ensemblesol[1] -# using StatsPlots -# plotly() -# plot(t, sol3flux_pestim.ensemblesol[1]) - -# function calculate_derivatives(dataset) -# x̂, time = dataset -# num_points = length(x̂) - -# # Initialize an array to store the derivative values. -# derivatives = similar(x̂) - -# for i in 2:(num_points - 1) -# # Calculate the first-order derivative using central differences. -# Δt_forward = time[i + 1] - time[i] -# Δt_backward = time[i] - time[i - 1] - -# derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) - -# derivatives[i] = derivative -# end - -# # Derivatives at the endpoints can be calculated using forward or backward differences. -# derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) -# derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) - -# return derivatives -# end - -# # Example usage: -# # dataset = [x̂, time] -# derivatives = calculate_derivatives(dataset) -# dataset[1] -# # Access derivative values at specific time points as needed. - -# # # 9,0.5 -# # 0.09894916260292887 -# # 0.09870335436072103 -# # 0.08398556878067913 -# # 0.10109070099105527 -# # 0.09122683737517055 -# # 0.08614958011892977 -# # mean(abs.(x̂ .- meanscurve1)) #0.017112298305523976 -# # mean(abs.(physsol1 .- meanscurve1)) #0.004038636894341354 -# # # 9,4(little worse) -# # mean(abs.(x̂ .- meanscurve1))#0.01800876370000113 -# # mean(abs.(physsol1 .- meanscurve1))#0.007285681280600875 -# # # 30,30 -# # mean(abs.(x̂ .- meanscurve1)) #0.10599926120358046 -# # mean(abs.(physsol1 .- meanscurve1)) #0.10375554193397989 -# # # 30,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.10160824458252521 -# # mean(abs.(physsol1 .- meanscurve1)) #0.09999942538357891 - -# # # ------------------------------------------------normale -# # # 9,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.0333356493928835 -# # mean(abs.(physsol1 .- meanscurve1)) #0.02721733876400459 -# # # 9,4(little worse) -# # mean(abs.(x̂ .- meanscurve1)) #0.020734206709433347 -# # mean(abs.(physsol1 .- meanscurve1)) #0.012502850740700212 -# # # 30,30 -# # mean(abs.(x̂ .- meanscurve1)) #0.10615859683094729 -# # mean(abs.(physsol1 .- meanscurve1)) #0.10508141153722575 -# # # 30,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.10833514946031565 -# # mean(abs.(physsol1 .- meanscurve1)) #0.10668470203219232 - -# # # 9,0.5 -# # 10.158108285475553 -# # 10.207234384538026 -# # 10.215000657664852 -# # 10.213817644016174 -# # 13.380030074088719 -# # 13.348906350967326 - -# # 6.952731422892041 - -# # # All losses -# # 10.161478523326277 -# # # L2 losses 1 -# # 9.33312996960278 -# # # L2 losses 2 -# # 10.217417241370631 - -# # mean([fhsamples1[i][26] for i in 500:1000]) #6.245045767509431 -# # p #6.283185307179586 -# # # 9,4 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.212522300650451 -# # # 30,30 -# # mean([fhsamples1[i][23] for i in 500:1000]) #35.328636809737695 -# # # 30,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #35.232963812125654 - -# # # ---------------------------------------normale -# # # 9,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.547771572198114 -# # p #6.283185307179586 -# # # 9,4 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.158906185002702 -# # # 30,30 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.210400972620185 -# # # 30,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.153845019454522 - -# # # ----------------more dataset normale ----------------------------- -# # # 9,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.271141178216537 -# # p #6.283185307179586 -# # # 9,4 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.241144692919369 -# # # 30,30 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.124480447973127 -# # # 30,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.07838011629903 - -# # # 9,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.016551602015599295 -# # mean(abs.(physsol1 .- meanscurve1)) #0.0021488618484224245 -# # # 9,4(little worse) -# # mean(abs.(x̂ .- meanscurve1)) #0.017022725082640747 -# # mean(abs.(physsol1 .- meanscurve1)) #0.004339761917100232 -# # # 30,30 -# # mean(abs.(x̂ .- meanscurve1)) #0.09668785317864312 -# # mean(abs.(physsol1 .- meanscurve1)) #0.09430712337543362 -# # # 30,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.09958118358974392 -# # mean(abs.(physsol1 .- meanscurve1)) #0.09717454226368502 - -# # # ----------------more dataset special ----------------------------- -# # # 9,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.284355334485365 -# # p #6.283185307179586 -# # # 9,4 -# # mean([fhsamples1[i][23] for i in 500:1000]) #6.259238106698602 -# # # 30,30 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.139808934336987 -# # # 30,0.5 -# # mean([fhsamples1[i][23] for i in 500:1000]) #29.03921327641226 - -# # # 9,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.016627231605546876 -# # mean(abs.(physsol1 .- meanscurve1)) #0.0020311429130039564 -# # # 9,4(little worse) -# # mean(abs.(x̂ .- meanscurve1)) #0.016650324577507352 -# # mean(abs.(physsol1 .- meanscurve1)) #0.0027537543411154677 -# # # 30,30 -# # mean(abs.(x̂ .- meanscurve1)) #0.09713187937270151 -# # mean(abs.(physsol1 .- meanscurve1)) #0.09317278450371556 -# # # 30,0.5 -# # mean(abs.(x̂ .- meanscurve1)) #0.09550234866855814 -# # mean(abs.(physsol1 .- meanscurve1)) #0.09317278450371556 - -# # using Plots, StatsPlots -# # plotly() - -# # --------------------------------------------------------- -# # # # Distribution abstract in wrapper, dataset Float64 -# # # 268.651 s (206393690 allocations: 388.71 GiB) -# # # 318.170551 seconds (206.29 M allocations: 388.453 GiB, 20.83% gc time) - -# # # # Above with dataset Real subtype -# # # 326.201 s (206327409 allocations: 388.42 GiB) -# # # 363.189370 seconds (206.25 M allocations: 387.975 GiB, 15.77% gc time) -# # # 306.171 s (206321277 allocations: 388.55 GiB) -# # # 356.180699 seconds (206.43 M allocations: 388.361 GiB, 13.77% gc time) - -# # # # Above with dataset AbstractFloat subtype -# # # 290.751187 seconds (205.94 M allocations: 387.955 GiB, 12.92% gc time) -# # # 296.319815 seconds (206.38 M allocations: 388.730 GiB, 12.69% gc time) - -# # # # ODEProblem float64 dtaset and vector distri inside -# # # 273.169 s (206128318 allocations: 388.40 GiB) -# # # 274.059531 seconds (205.91 M allocations: 387.953 GiB, 12.77% gc time) - -# # # # Dataset float64 inside and vector distri outsude -# # # 333.603 s (206251143 allocations: 388.41 GiB) -# # # 373.377222 seconds (206.11 M allocations: 387.968 GiB, 13.25% gc time) -# # # 359.745 s (206348301 allocations: 388.41 GiB) -# # # 357.813114 seconds (206.31 M allocations: 388.354 GiB, 13.54% gc time) - -# # # # Dataset float64 inside and vector distri inside -# # # 326.437 s (206253571 allocations: 388.41 GiB) -# # # 290.334083 seconds (205.92 M allocations: 387.954 GiB, 13.82% gc time) - -# # # # current setting -# # # 451.304 s (206476927 allocations: 388.43 GiB) -# # # 384.532732 seconds (206.22 M allocations: 387.976 GiB, 13.17% gc time) -# # # 310.223 s (206332558 allocations: 388.63 GiB) -# # # 344.243889 seconds (206.34 M allocations: 388.409 GiB, 13.84% gc time) -# # # 357.457737 seconds (206.66 M allocations: 389.064 GiB, 18.16% gc time) - -# # # # shit setup -# # # 325.595 s (206283732 allocations: 388.41 GiB) -# # # 334.248753 seconds (206.06 M allocations: 387.964 GiB, 12.60% gc time) -# # # 326.011 s (206370857 allocations: 388.56 GiB) -# # # 327.203339 seconds (206.29 M allocations: 388.405 GiB, 12.92% gc time) - -# # # # in wrapper Distribution prior, insiade FLOAT64 DATASET -# # # 325.158167 seconds (205.97 M allocations: 387.958 GiB, 15.07% gc time) -# # # 429.536 s (206476324 allocations: 388.43 GiB) -# # # 527.364 s (206740343 allocations: 388.58 GiB) - -# # # # wrapper Distribtuion, inside Float64 -# # # 326.017 s (206037971 allocations: 387.96 GiB) -# # # 347.424730 seconds (206.45 M allocations: 388.532 GiB, 12.92% gc time) - -# # # 439.047568 seconds (284.24 M allocations: 392.598 GiB, 15.25% gc time, 14.36% compilation time: 0% of which was recompilation) -# # # 375.472142 seconds (206.40 M allocations: 388.529 GiB, 14.93% gc time) -# # # 374.888820 seconds (206.34 M allocations: 388.346 GiB, 14.09% gc time) -# # # 363.719611 seconds (206.39 M allocations: 388.581 GiB, 15.08% gc time) -# # # # inside Distribtion, instide Float64 -# # # 310.238 s (206324249 allocations: 388.53 GiB) -# # # 308.991494 seconds (206.34 M allocations: 388.549 GiB, 14.01% gc time) -# # # 337.442 s (206280712 allocations: 388.36 GiB) -# # # 299.983096 seconds (206.29 M allocations: 388.512 GiB, 17.14% gc time) - -# # # 394.924357 seconds (206.27 M allocations: 388.337 GiB, 23.68% gc time) -# # # 438.204179 seconds (206.39 M allocations: 388.470 GiB, 23.84% gc time) -# # # 376.626914 seconds (206.46 M allocations: 388.693 GiB, 18.72% gc time) -# # # 286.863795 seconds (206.14 M allocations: 388.370 GiB, 18.80% gc time) -# # # 285.556929 seconds (206.22 M allocations: 388.371 GiB, 17.04% gc time) -# # # 291.471662 seconds (205.96 M allocations: 388.068 GiB, 19.85% gc time) - -# # # 495.814341 seconds (284.62 M allocations: 392.622 GiB, 12.56% gc time, 10.96% compilation time: 0% of which was recompilation) -# # # 361.530617 seconds (206.36 M allocations: 388.526 GiB, 14.98% gc time) -# # # 348.576065 seconds (206.22 M allocations: 388.337 GiB, 15.01% gc time) -# # # 374.575609 seconds (206.45 M allocations: 388.586 GiB, 14.65% gc time) -# # # 314.223008 seconds (206.23 M allocations: 388.411 GiB, 14.63% gc time) - -# # PROBLEM-3 LOTKA VOLTERRA EXAMPLE [WIP] (WITH PARAMETER ESTIMATION)(will be put in tutorial page) -# function lotka_volterra(u, p, t) -# # Model parameters. -# α, β, γ, δ = p -# # Current state. -# x, y = u - -# # Evaluate differential equations. -# dx = (α - β * y) * x # prey -# dy = (δ * x - γ) * y # predator - -# return [dx, dy] -# end - -# u0 = [1.0, 1.0] -# p = [1.5, 1.0, 3.0, 1.0] -# tspan = (0.0, 6.0) -# prob = ODEProblem(lotka_volterra, u0, tspan, p) -# solution = solve(prob, Tsit5(); saveat = 0.05) - -# as = reduce(hcat, solution.u) -# as[1, :] -# # Plot simulation. -# time = solution.t -# u = hcat(solution.u...) -# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct -# x = u[1, :] + 0.5 * randn(length(u[1, :])) -# y = u[2, :] + 0.5 * randn(length(u[1, :])) -# dataset = [x[1:50], y[1:50], time[1:50]] -# # scatter!(time, [x, y]) -# # scatter!(dataset[3], [dataset[2], dataset[1]]) - -# # NN has 2 outputs as u -> [dx,dy] -# chainlux1 = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), -# Lux.Dense(6, 2)) -# chainflux1 = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) - -# # fh_mcmc_chainflux1, fhsamplesflux1, fhstatsflux1 = ahmc_bayesian_pinn_ode(prob, chainflux1, -# # dataset = dataset, -# # draw_samples = 1000, -# # l2std = [ -# # 0.05, -# # 0.05, -# # ], -# # phystd = [ -# # 0.05, -# # 0.05, -# # ], -# # priorsNNw = (0.0, -# # - -# # 3.0)) - -# # check if NN output is more than 1 -# # numoutput = size(luxar[1])[1] -# # if numoutput > 1 -# # # Initialize a vector to store the separated outputs for each output dimension -# # output_matrices = [Vector{Vector{Float32}}() for _ in 1:numoutput] - -# # # Loop through each element in the `as` vector -# # for element in as -# # for i in 1:numoutput -# # push!(output_matrices[i], element[i, :]) # Append the i-th output (i-th row) to the i-th output_matrices -# # end -# # end - -# # ensemblecurves = Vector{}[] -# # for r in 1:numoutput -# # br = hcat(output_matrices[r]...)' -# # ensemblecurve = prob.u0[r] .+ -# # [Particles(br[:, i]) for i in 1:length(t)] .* -# # (t .- prob.tspan[1]) -# # push!(ensemblecurves, ensemblecurve) -# # end - -# # else -# # # ensemblecurve = prob.u0 .+ -# # # [Particles(reduce(vcat, luxar)[:, i]) for i in 1:length(t)] .* -# # # (t .- prob.tspan[1]) -# # print("yuh") -# # end - -# # fhsamplesflux2 -# # nnparams = length(init1) -# # estimnnparams = [Particles(reduce(hcat, fhsamplesflux2)[i, :]) for i in 1:nnparams] -# # ninv=4 -# # estimated_params = [Particles(reduce(hcat, fhsamplesflux2[(end - ninv + 1):end])[i, :]) -# # for i in (nnparams + 1):(nnparams + ninv)] -# # output_matrices[r] -# # br = hcat(output_matrices[r]...)' - -# # br[:, 1] - -# # [Particles(br[:, i]) for i in 1:length(t)] -# # prob.u0 -# # [Particles(br[:, i]) for i in 1:length(t)] .* -# # (t .- prob.tspan[1]) - -# # ensemblecurve = prob.u0[r] .+ -# # [Particles(br[:, i]) for i in 1:length(t)] .* -# # (t .- prob.tspan[1]) -# # push!(ensemblecurves, ensemblecurve) - -# using StatsPlots -# plotly() -# plot(t, ensemblecurve) -# plot(t, ensemblecurves[1]) -# plot!(t, ensemblecurves[2]) -# ensemblecurve -# ensemblecurves[1] -# fh_mcmc_chainflux2, fhsamplesflux2, fhstatsflux2 = ahmc_bayesian_pinn_ode(prob, chainflux1, -# dataset = dataset, -# draw_samples = 1000, -# l2std = [ -# 0.05, -# 0.05, -# ], -# phystd = [ -# 0.05, -# 0.05, -# ], -# priorsNNw = (0.0, -# 3.0), -# param = [ -# Normal(1.5, -# 0.5), -# Normal(1.2, -# 0.5), -# Normal(3.3, -# 0.5), -# Normal(1.4, -# 0.5), -# ], progress = true) - -# alg = NeuralPDE.BNNODE(chainflux1, -# dataset = dataset, -# draw_samples = 1000, -# l2std = [ -# 0.05, -# 0.05, -# ], -# phystd = [ -# 0.05, -# 0.05, -# ], -# priorsNNw = (0.0, -# 3.0), -# param = [ -# Normal(4.5, -# 5), -# Normal(7, -# 2), -# Normal(5, -# 2), -# Normal(-4, -# 6), -# ], -# n_leapfrog = 30, progress = true) - -# sol3flux_pestim = solve(prob, alg) - -# # OG PARAM VALUES -# [1.5, 1.0, 3.0, 1.0] -# # less -# # [1.34, 7.51, 2.54, -2.55] -# # better -# # [1.48, 0.993, 2.77, 0.954] - -# sol3flux_pestim.es -# sol3flux_pestim.estimated_ode_params -# # fh_mcmc_chainlux1, fhsampleslux1, fhstatslux1 = ahmc_bayesian_pinn_ode(prob, chainlux1, -# # dataset = dataset, -# # draw_samples = 1000, -# # l2std = [0.05, 0.05], -# # phystd = [ -# # 0.05, -# # 0.05, -# # ], -# # priorsNNw = (0.0, -# # 3.0)) - -# # fh_mcmc_chainlux2, fhsampleslux2, fhstatslux2 = ahmc_bayesian_pinn_ode(prob, chainlux1, -# # dataset = dataset, -# # draw_samples = 1000, -# # l2std = [0.05, 0.05], -# # phystd = [ -# # 0.05, -# # 0.05, -# # ], -# # priorsNNw = (0.0, -# # 3.0), -# # param = [ -# # Normal(1.5, 0.5), -# # Normal(1.2, 0.5), -# # Normal(3.3, 0.5), -# # Normal(1.4, 0.5), -# # ]) - -# init1, re1 = destructure(chainflux1) -# θinit, st = Lux.setup(Random.default_rng(), chainlux1) -# # PLOT testing points -# t = time -# p = prob.p -# collect(Float64, vcat(ComponentArrays.ComponentArray(θinit))) -# collect(Float64, ComponentArrays.ComponentArray(θinit)) -# # Mean of last 1000 sampled parameter's curves(flux and lux chains)[Ensemble predictions] -# out = re1.([fhsamplesflux1[i][1:68] for i in 500:1000]) -# yu = [out[i](t') for i in eachindex(out)] - -# function getensemble(yu, num_models) -# num_rows, num_cols = size(yu[1]) -# row_means = zeros(Float32, num_rows, num_cols) -# for i in 1:num_models -# row_means .+= yu[i] -# end -# row_means ./ num_models -# end -# fluxmean = getensemble(yu, length(out)) -# meanscurve1_1 = prob.u0 .+ (t' .- prob.tspan[1]) .* fluxmean -# mean(abs.(u .- meanscurve1_1)) - -# plot!(t, physsol1) -# @test mean(abs2.(x̂ .- meanscurve1_1)) < 2e-2 -# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 -# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 -# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 - -# out = re1.([fhsamplesflux2[i][1:68] for i in 500:1000]) -# yu = collect(out[i](t') for i in eachindex(out)) -# fluxmean = getensemble(yu, length(out)) -# meanscurve1_2 = prob.u0 .+ (t' .- prob.tspan[1]) .* fluxmean -# mean(abs.(u .- meanscurve1_2)) - -# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 -# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 -# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 -# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 - -# θ = [vector_to_parameters(fhsampleslux1[i][1:(end - 4)], θinit) for i in 500:1000] -# luxar = [chainlux1(t', θ[i], st)[1] for i in 1:500] -# luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] -# meanscurve2_1 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean - -# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 -# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 -# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 -# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 - -# θ = [vector_to_parameters(fhsampleslux2[i][1:(end - 4)], θinit) for i in 500:1000] -# luxar = [chainlux1(t', θ[i], st)[1] for i in 1:500] -# luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] -# meanscurve2_2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean - -# @test mean(abs2.(x̂ .- meanscurve1)) < 2e-2 -# @test mean(abs2.(physsol1 .- meanscurve1)) < 2e-2 -# @test mean(abs2.(x̂ .- meanscurve2)) < 3e-3 -# @test mean(abs2.(physsol1 .- meanscurve2)) < 2e-3 - -# # # ESTIMATED ODE PARAMETERS (NN1 AND NN2) -# @test abs(p - mean([fhsamplesflux2[i][69] for i in 500:1000])) < 0.1 * p[1] -# @test abs(p - mean([fhsampleslux2[i][69] for i in 500:1000])) < 0.2 * p[1] - -# # @test abs(p - mean([fhsamplesflux2[i][70] for i in 500:1000])) < 0.1 * p[2] -# # @test abs(p - mean([fhsampleslux2[i][70] for i in 500:1000])) < 0.2 * p[2] - -# # @test abs(p - mean([fhsamplesflux2[i][71] for i in 500:1000])) < 0.1 * p[3] -# # @test abs(p - mean([fhsampleslux2[i][71] for i in 500:1000])) < 0.2 * p[3] - -# # @test abs(p - mean([fhsamplesflux2[i][72] for i in 500:1000])) < 0.1 * p[4] -# # @test abs(p - mean([fhsampleslux2[i][72] for i in 500:1000])) < 0.2 * p[4] - -# # fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainlux1, -# # dataset = dataset, -# # draw_samples = 1000, -# # l2std = [0.05, 0.05], -# # phystd = [0.05, 0.05], -# # priorsNNw = (0.0, 3.0), -# # param = [ -# # Normal(1.5, 0.5), -# # Normal(1.2, 0.5), -# # Normal(3.3, 0.5), -# # Normal(1.4, 0.5), -# # ], autodiff = true) - -# # fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chainlux1, -# # dataset = dataset, -# # draw_samples = 1000, -# # l2std = [0.05, 0.05], -# # phystd = [0.05, 0.05], -# # priorsNNw = (0.0, 3.0), -# # param = [ -# # Normal(1.5, 0.5), -# # Normal(1.2, 0.5), -# # Normal(3.3, 0.5), -# # Normal(1.4, 0.5), -# # ], nchains = 2) - -# # NOTES (WILL CLEAR LATER) -# # -------------------------------------------------------------------------------------------- -# # Hamiltonian energy must be lowest(more paramters the better is it to map onto them) -# # full better than L2 and phy individual(test) -# # in mergephys more points after training points is better from 20->40 -# # does consecutive runs bceome better? why?(plot 172)(same chain maybe) -# # does density of points in timespan matter dataset vs internal timespan?(plot 172)(100+0.01) -# # when training from 0-1 and phys from 1-5 with 1/150 simple nn slow,but bigger nn faster decrease in Hmailtonian -# # bigger time interval more curves to adapt to only more parameters adapt to that, better NN architecture -# # higher order logproblems solve better -# # repl up up are same instances? but reexecute calls are new? - -# #Compare results against paper example -# # Lux chains support (DONE) -# # fix predictions for odes depending upon 1,p in f(u,p,t)(DONE) -# # lotka volterra learn curve beyond l2 losses(L2 losses determine accuracy of parameters)(parameters cant run free ∴ L2 interval only) -# # check if prameters estimation works(YES) -# # lotka volterra parameters estimate (DONE) - -# using NeuralPDE, Lux, Flux, Optimization, OptimizationOptimJL -# import ModelingToolkit: Interval -# using Plots, StatsPlots -# plotly() -# # Profile.init() - -# @parameters x y -# @variables u(..) -# Dxx = Differential(x)^2 -# Dyy = Differential(y)^2 - -# # 2D PDE -# eq = Dxx(u(x, y)) + Dyy(u(x, y)) ~ -sin(pi * x) * sin(pi * y) - -# # Boundary conditions -# bcs = [u(0, y) ~ 0.0, u(1, y) ~ 0.0, -# u(x, 0) ~ 0.0, u(x, 1) ~ 0.0] -# # Space and time domains -# domains = [x ∈ Interval(0.0, 1.0), -# y ∈ Interval(0.0, 1.0)] - -# # Neural network -# dim = 2 # number of dimensions -# chain = Flux.Chain(Flux.Dense(dim, 16, Lux.σ), Flux.Dense(16, 16, Lux.σ), Flux.Dense(16, 1)) -# θ, re = destructure(chain) -# # Discretization -# dx = 0.05 -# discretization = PhysicsInformedNN(chain, GridTraining(dx)) - -# @named pde_system = PDESystem(eq, bcs, domains, [x, y], [u(x, y)]) - -# pinnrep = symbolic_discretize(pde_system, discretization) -# typeof(pinnrep.phi) -# typeof(pinnrep.phi) -# typeof(re) -# pinnrep.phi([1, 2], θ) - -# typeof(θ) - -# print(pinnrep) -# pinnrep.eqs -# pinnrep.bcs -# pinnrep.domains -# pinnrep.eq_params -# pinnrep.defaults -# print(pinnrep.default_p) -# pinnrep.param_estim -# print(pinnrep.additional_loss) -# pinnrep.adaloss -# pinnrep.depvars -# pinnrep.indvars -# pinnrep.dict_depvar_input -# pinnrep.dict_depvars -# pinnrep.dict_indvars -# print(pinnrep.logger) -# pinnrep.multioutput -# pinnrep.iteration -# pinnrep.init_params -# pinnrep.flat_init_params -# pinnrep.phi -# pinnrep.derivative -# pinnrep.strategy -# pinnrep.pde_indvars -# pinnrep.bc_indvars -# pinnrep.pde_integration_vars -# pinnrep.bc_integration_vars -# pinnrep.integral -# pinnrep.symbolic_pde_loss_functions -# pinnrep.symbolic_bc_loss_functions -# pinnrep.loss_functions - -# # = discretize(pde_system, discretization) -# prob = symbolic_discretize(pde_system, discretization) -# # "The boundary condition loss functions" -# sum([prob.loss_functions.bc_loss_functions[i](θ) for i in eachindex(1:4)]) -# sum([prob.loss_functions.pde_loss_functions[i](θ) for i in eachindex(1)]) - -# prob.loss_functions.full_loss_function(θ, 32) - -# prob.loss_functions.bc_loss_functions[1](θ) - -# prob.loss_functions.bc_loss_functions -# prob.loss_functions.full_loss_function -# prob.loss_functions.additional_loss_function -# prob.loss_functions.pde_loss_functions - -# 1.3953060473003345 + 1.378102161087438 + 1.395376727128639 + 1.3783868705075002 + -# 0.22674532775196876 -# # "The PDE loss functions" -# prob.loss_functions.pde_loss_functions -# prob.loss_functions.pde_loss_functions[1](θ) -# # "The full loss function, combining the PDE and boundary condition loss functions.This is the loss function that is used by the optimizer." -# prob.loss_functions.full_loss_function(θ, nothing) -# prob.loss_functions.full_loss_function(θ, 423423) - -# # "The wrapped `additional_loss`, as pieced together for the optimizer." -# prob.loss_functions.additional_loss_function -# # "The pre-data version of the PDE loss function" -# prob.loss_functions.datafree_pde_loss_functions -# # "The pre-data version of the BC loss function" -# prob.loss_functions.datafree_bc_loss_functions - -# using Random -# θ, st = Lux.setup(Random.default_rng(), chain) -# #Optimizer -# opt = OptimizationOptimJL.BFGS() - -# #Callback function -# callback = function (p, l) -# println("Current loss is: $l") -# return false -# end - -# res = Optimization.solve(prob, opt, callback = callback, maxiters = 1000) -# phi = discretization.phi - -# # ------------------------------------------------ -# using NeuralPDE, Lux, ModelingToolkit, Optimization, OptimizationOptimJL, OrdinaryDiffEq, -# Plots -# import ModelingToolkit: Interval, infimum, supremum -# @parameters t, σ_, β, ρ -# @variables x(..), y(..), z(..) -# Dt = Differential(t) -# eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), -# Dt(y(t)) ~ x(t) * (ρ - z(t)) - y(t), -# Dt(z(t)) ~ x(t) * y(t) - β * z(t)] - -# bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] -# domains = [t ∈ Interval(0.0, 1.0)] -# dt = 0.01 - -# input_ = length(domains) -# n = 8 -# chain1 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, 1)) -# chain2 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, 1)) -# chain3 = Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, n, Lux.σ), -# Lux.Dense(n, 1)) - -# function lorenz!(du, u, p, t) -# du[1] = 10.0 * (u[2] - u[1]) -# du[2] = u[1] * (28.0 - u[3]) - u[2] -# du[3] = u[1] * u[2] - (8 / 3) * u[3] -# end - -# u0 = [1.0; 0.0; 0.0] -# tspan = (0.0, 1.0) -# prob = ODEProblem(lorenz!, u0, tspan) -# sol = solve(prob, Tsit5(), dt = 0.1) -# ts = [infimum(d.domain):dt:supremum(d.domain) for d in domains][1] -# function getData(sol) -# data = [] -# us = hcat(sol(ts).u...) -# ts_ = hcat(sol(ts).t...) -# return [us, ts_] -# end -# data = getData(sol) - -# (u_, t_) = data -# len = length(data[2]) - -# depvars = [:x, :y, :z] -# function additional_loss(phi, θ, p) -# return sum(sum(abs2, phi[i](t_, θ[depvars[i]]) .- u_[[i], :]) / len for i in 1:1:3) -# end - -# discretization = NeuralPDE.PhysicsInformedNN([chain1, chain2, chain3], -# NeuralPDE.GridTraining(dt), -# param_estim = false, -# additional_loss = additional_loss) -# @named pde_system = PDESystem(eqs, bcs, domains, [t], [x(t), y(t), z(t)], [σ_, ρ, β], -# defaults = Dict([p .=> 1.0 for p in [σ_, ρ, β]])) -# prob = NeuralPDE.discretize(pde_system, discretization) -# callback = function (p, l) -# println("Current loss is: $l") -# return false -# end -# res = Optimization.solve(prob, BFGS(); callback = callback, maxiters = 5000) -# p_ = res.u[(end - 2):end] # p_ = [9.93, 28.002, 2.667] - -# minimizers = [res.u.depvar[depvars[i]] for i in 1:3] -# ts = [infimum(d.domain):(dt / 10):supremum(d.domain) for d in domains][1] -# u_predict = [[discretization.phi[i]([t], minimizers[i])[1] for t in ts] for i in 1:3] -# plot(sol) -# plot!(ts, u_predict, label = ["x(t)" "y(t)" "z(t)"]) - -# discretization.multioutput -# discretization.chain -# discretization.strategy -# discretization.init_params -# discretization.phi -# discretization.derivative -# discretization.param_estim -# discretization.additional_loss -# discretization.adaptive_loss -# discretization.logger -# discretization.log_options -# discretization.iteration -# discretization.self_increment -# discretization.multioutput -# discretization.kwargs - -# struct BNNODE1{P <: Vector{<:Distribution}} -# chain::Any -# Kernel::Any -# draw_samples::UInt32 -# priorsNNw::Tuple{Float64, Float64} -# param::P -# l2std::Vector{Float64} -# phystd::Vector{Float64} - -# function BNNODE1(chain, Kernel; draw_samples = 2000, priorsNNw = (0.0, 3.0), param = [], -# l2std = [0.05], phystd = [0.05]) -# BNNODE1(chain, Kernel, draw_samples, priorsNNw, param, l2std, phystd) -# end -# end - -# struct BNNODE3{C, K, P <: Union{Any, Vector{<:Distribution}}} -# chain::C -# Kernel::K -# draw_samples::UInt32 -# priorsNNw::Tuple{Float64, Float64} -# param::P -# l2std::Vector{Float64} -# phystd::Vector{Float64} - -# function BNNODE3(chain, Kernel; draw_samples = 2000, priorsNNw = (0.0, 3.0), param = [], -# l2std = [0.05], phystd = [0.05]) -# new{typeof(chain), typeof(Kernel), typeof(param)}(chain, Kernel, draw_samples, -# priorsNNw, param, l2std, phystd) -# end -# end -# linear_analytic = (u0, p, t) -> u0 + sin(2 * π * t) / (2 * π) -# linear = (u, p, t) -> cos(2 * π * t) -# tspan = (0.0, 2.0) -# u0 = 0.0 -# prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan) - -# ta = range(tspan[1], tspan[2], length = 300) -# u = [linear_analytic(u0, nothing, ti) for ti in ta] -# sol1 = solve(prob, Tsit5()) - -# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct -# x̂ = collect(Float64, Array(u) + 0.02 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂[1:100], time[1:100]] - -# # Call BPINN, create chain -# chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) -# chainlux = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) -# HMC -# solve(prob, BNNODE(chainflux, HMC)) -# BNNODE1(chainflux, HMC, 2000) - -# draw_samples = 2000 -# priorsNNw = (0.0, 3.0) -# param = [] -# l2std = [0.05] -# phystd = [0.05] -# @time BNNODE3(chainflux, HMC, draw_samples = 2000, priorsNNw = (0.0, 3.0), -# param = [nothing], -# l2std = [0.05], phystd = [0.05]) -# typeof(Nothing) <: Vector{<:Distribution} -# Nothing <: Distribution -# {UnionAll} <: Distribution -# @time [Nothing] -# typeof([Nothing]) -# @time [1] - -# function test1(sum; c = 23, d = 32) -# return sum + c + d -# end -# function test(a, b; c, d) -# return test1(a + b, c, d) -# end - -# test(2, 2) - -# struct BNNODE3{C, K, P <: Union{Vector{Nothing}, Vector{<:Distribution}}} -# chain::C -# Kernel::K -# draw_samples::Int64 -# priorsNNw::Tuple{Float64, Float64} -# param::P -# l2std::Vector{Float64} -# phystd::Vector{Float64} - -# function BNNODE3(chain, Kernel; draw_samples, -# priorsNNw, param = [nothing], l2std, phystd) -# new{typeof(chain), typeof(Kernel), typeof(param)}(chain, -# Kernel, -# draw_samples, -# priorsNNw, -# param, l2std, -# phystd) -# end -# end - -# function solve1(prob::DiffEqBase.AbstractODEProblem, alg::BNNODE3; -# dataset = [nothing], dt = 1 / 20.0, -# init_params = nothing, nchains = 1, -# autodiff = false, Integrator = Leapfrog, -# Adaptor = StanHMCAdaptor, targetacceptancerate = 0.8, -# Metric = DiagEuclideanMetric, jitter_rate = 3.0, -# tempering_rate = 3.0, max_depth = 10, Δ_max = 1000, -# n_leapfrog = 10, δ = 0.65, λ = 0.3, progress = true, -# verbose = false) -# chain = alg.chain -# l2std = alg.l2std -# phystd = alg.phystd -# priorsNNw = alg.priorsNNw -# Kernel = alg.Kernel -# draw_samples = alg.draw_samples - -# param = alg.param == [nothing] ? [] : alg.param -# mcmcchain, samples, statistics = ahmc_bayesian_pinn_ode(prob, chain, dataset = dataset, -# draw_samples = draw_samples, -# init_params = init_params, -# physdt = dt, l2std = l2std, -# phystd = phystd, -# priorsNNw = priorsNNw, -# param = param, -# nchains = nchains, -# autodiff = autodiff, -# Kernel = Kernel, -# Integrator = Integrator, -# Adaptor = Adaptor, -# targetacceptancerate = targetacceptancerate, -# Metric = Metric, -# jitter_rate = jitter_rate, -# tempering_rate = tempering_rate, -# max_depth = max_depth, -# Δ_max = Δ_max, -# n_leapfrog = n_leapfrog, δ = δ, -# λ = λ, progress = progress, -# verbose = verbose) -# end - -# linear_analytic = (u0, p, t) -> u0 + sin(2 * π * t) / (2 * π) -# linear = (u, p, t) -> cos(2 * π * t) -# tspan = (0.0, 2.0) -# u0 = 0.0 -# prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), u0, tspan) - -# ta = range(tspan[1], tspan[2], length = 300) -# u = [linear_analytic(u0, nothing, ti) for ti in ta] -# # sol1 = solve(prob, Tsit5()) - -# # BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct -# x̂ = collect(Float64, Array(u) + 0.02 * randn(size(u))) -# time = vec(collect(Float64, ta)) -# dataset = [x̂[1:100], time[1:100]] - -# # Call BPINN, create chain -# chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) -# chainlux = Lux.Chain(Lux.Dense(1, 7, tanh), Lux.Dense(7, 1)) -# HMC - -# solve1(prob, a) -# a = BNNODE3(chainflux, HMC, draw_samples = 2000, -# priorsNNw = (0.0, 3.0), -# l2std = [0.05], phystd = [0.05]) - -# Define Lotka-Volterra model. -function lotka_volterra1(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end - -u0 = [1.0, 1.0] -p = [1.5, 1.0, 3.0, 1.0] -tspan = (0.0, 6.0) -prob = ODEProblem(lotka_volterra1, u0, tspan, p) -solution = solve(prob, Tsit5(); saveat = 0.05) - -as = reduce(hcat, solution.u) -as[1, :] -# Plot simulation. -time = solution.t -u = hcat(solution.u...) -# BPINN AND TRAINING DATASET CREATION, NN create, Reconstruct -x = u[1, :] + 0.5 * randn(length(u[1, :])) -y = u[2, :] + 0.5 * randn(length(u[1, :])) -dataset = [x[1:50], y[1:50], time[1:50]] -# scatter!(time, [x, y]) -# scatter!(dataset[3], [dataset[2], dataset[1]]) - -# NN has 2 outputs as u -> [dx,dy] -chainlux1 = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), - Lux.Dense(6, 2)) -chainflux1 = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) - -fh_mcmc_chainflux1, fhsamplesflux1, fhstatsflux1 = ahmc_bayesian_pinn_ode(prob, chainflux1, - dataset = dataset, - draw_samples = 1000, - l2std = [ - 0.05, - 0.05, - ], - phystd = [ - 0.05, - 0.05, - ], - priorsNNw = (0.0, 3.0), progress = true) -ahmc_bayesian_pinn_ode(prob, chainflux1, - dataset = dataset, - draw_samples = 1000, - l2std = [ - 0.05, - 0.05, - ], - phystd = [ - 0.05, - 0.05, - ], - priorsNNw = (0.0, 3.0), progress = true) - -# 2×171 Matrix{Float64}: -# -0.5 -0.518956 -0.529639 … -1.00266 -1.01049 -# 2.0 1.97109 1.92747 0.42619 0.396335 - -# 2-element Vector{Float64}: -# -119451.94949911036 -# -128543.23714618056 - -# alg = NeuralPDE.BNNODE(chainflux1, -# dataset = dataset, -# draw_samples = 1000, -# l2std = [ -# 0.05, -# 0.05, -# ], -# phystd = [ -# 0.05, -# 0.05, -# ], -# priorsNNw = (0.0, -# 3.0), -# param = [ -# Normal(4.5, -# 5), -# Normal(7, -# 2), -# Normal(5, -# 2), -# Normal(-4, -# 6), -# ], -# n_leapfrog = 30, progress = true) - -# sol3flux_pestim = solve(prob, alg) - -# ---------------------------------------------- -# original paper implementation -# 25 points -run1 #7.70593 Particles{Float64, 1} -run2 #6.66347 Particles{Float64, 1} -run3 #6.84827 Particles{Float64, 1} - -# 50 points -run1 #7.83577 Particles{Float64, 1} -run2 #6.49477 Particles{Float64, 1} -run3 #6.47421 Particles{Float64, 1} - -# 100 points -run1 #5.96604 Particles{Float64, 1} -run2 #6.05432 Particles{Float64, 1} -run3 #6.08856 Particles{Float64, 1} - -# Full likelihood(uses total variation regularized differentiation) -# 25 points -run1 #6.41722 Particles{Float64, 1} -run2 #6.42782 Particles{Float64, 1} -run3 #6.42782 Particles{Float64, 1} - -# 50 points -run1 #5.71268 Particles{Float64, 1} -run2 #5.74599 Particles{Float64, 1} -run3 #5.74599 Particles{Float64, 1} - -# 100 points -run1 #6.59097 Particles{Float64, 1} -run2 #6.62813 Particles{Float64, 1} -run3 #6.62813 Particles{Float64, 1} - -using Plots, StatsPlots -function lotka_volterra(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end - -# initial-value problem. -u0 = [1.0, 1.0] -p = [1.5, 1.0, 3.0, 1.0] -tspan = (0.0, 6.0) -prob = ODEProblem(lotka_volterra, u0, tspan, p) - -# Plot simulation. - -solution = solve(prob, Tsit5(); saveat = 0.05) -plot(solve(prob, Tsit5())) - -# Dataset creation for parameter estimation -time = solution.t -u = hcat(solution.u...) -x = u[1, :] + 0.5 * randn(length(u[1, :])) -y = u[2, :] + 0.5 * randn(length(u[1, :])) -dataset = [x, y, time] - -# Neural Networks must have 2 outputs as u -> [dx,dy] in function lotka_volterra() -chainflux = Flux.Chain(Flux.Dense(1, 6, tanh), Flux.Dense(6, 6, tanh), Flux.Dense(6, 2)) |> - Flux.f64 - -chainlux = Lux.Chain(Lux.Dense(1, 6, Lux.tanh), Lux.Dense(6, 6, Lux.tanh), Lux.Dense(6, 2)) - -alg1 = NeuralPDE.BNNODE(chainflux, - dataset = dataset, - draw_samples = 1000, - l2std = [ - 0.01, - 0.01, - ], - phystd = [ - 0.01, - 0.01, - ], - priorsNNw = (0.0, - 3.0), - param = [ - LogNormal(1.5, - 0.5), - LogNormal(1.2, - 0.5), - LogNormal(3.3, - 1), - LogNormal(1.4, - 1)], - n_leapfrog = 30, progress = true) - -sol_flux_pestim = solve(prob, alg1) - -# Dataset not needed as we are solving the equation with ideal parameters -alg2 = NeuralPDE.BNNODE(chainlux, - draw_samples = 1000, - l2std = [ - 0.05, - 0.05, - ], - phystd = [ - 0.05, - 0.05, - ], - priorsNNw = (0.0, - 3.0), - n_leapfrog = 30, progress = true) - -sol_lux = solve(prob, alg2) - -#testing timepoints must match keyword arg `saveat`` timepoints of solve() call -t = collect(Float64, prob.tspan[1]:(1 / 50.0):prob.tspan[2]) - -# plotting solution for x,y for chain_flux -plot(t, sol_flux_pestim.ensemblesol[1]) -plot!(t, sol_flux_pestim.ensemblesol[2]) - -plot(sol_flux_pestim.ens1mblesol[1]) -plot!(sol_flux_pestim.ensemblesol[2]) - -# estimated ODE parameters by .estimated_ode_params, weights and biases by .estimated_nn_params -sol_flux_pestim.estimated_nn_params -sol_flux_pestim.estimated_ode_params - -# plotting solution for x,y for chain_lux -plot(t, sol_lux.ensemblesol[1]) -plot!(t, sol_lux.ensemblesol[2]) - -# estimated weights and biases by .estimated_nn_params for chain_lux -sol_lux.estimated_nn_params - -# # ----------------------------------stats----------------------------- -# # ---------------------------- -# # ----------------------------- -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:04:47 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:03:38 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:04:12 -# # -------------------------- -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:05:09 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:04:47 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:04:25 -# # -------------- -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:06:47 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:05:54 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:05:46 -# # ------------------------ -# # ----------------------- -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -882.2934218498742 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:04:06 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -882.2934218498742 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:03:32 - -# physics Logpdf is : -15740.509286661572 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -882.2934218498742 -# L2loss2 Logpdf is : -3118.0639515039957 -# Sampling 100%|███████████████████████████████| Time: 0:03:01 -# # -------------------------- -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1411.1717435511828 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:04:02 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1411.1717435511828 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:04:08 - -# physics Logpdf is : -18864.79640643607 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -1411.1717435511828 -# L2loss2 Logpdf is : -6242.351071278482 -# Sampling 100%|███████████████████████████████| Time: 0:04:15 -# # ---------------------------- -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -3240.067149411982 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:05:37 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -3240.067149411982 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:06:02 - -# physics Logpdf is : -25119.77191296288 -# prior Logpdf is : -139.5069300318621 -# L2lossData Logpdf is : -3240.067149411982 -# L2loss2 Logpdf is : -12497.32657780532 -# Sampling 100%|███████████████████████████████| Time: 0:06:13 - -using NeuralPDE, Lux, ModelingToolkit, Optimization, OptimizationOptimJL -import ModelingToolkit: Interval, infimum, supremum - -using NeuralPDE, Flux, OptimizationOptimisers - -function diffeq(u, p, t) - u1, u2 = u - return [u2, p[1] + p[2] * sin(u1) + p[3] * u2] -end -p = [5, -10, -1.7] -u0 = [-1.0, 7.0] -tspan = (0.0, 10.0) -prob = ODEProblem(ODEFunction(diffeq), u0, tspan, p) - -chainnew = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), Flux.Dense(8, 2)) |> - Flux.f64 - -opt = OptimizationOptimisers.Adam(0.1) -opt = Optimisers.ADAGrad(0.1) -opt = Optimisers.AdaMax(0.01) -algnew = NeuralPDE.NNODE(chainnew, opt) -solution_new = solve(prob, algnew, verbose = true, - abstol = 1e-10, maxiters = 7000) -u = reduce(hcat, solution_new.u) -plot(solution_new.t, u[1, :]) -plot!(solution_new.t, u[2, :]) - -algnew = NeuralPDE.BNNODE(chainnew, draw_samples = 200, - n_leapfrog = 30, progress = true) -solution_new = solve(prob, algnew) - -@parameters t -@variables u1(..), u2(..) -D = Differential(t) -eq = [D(u1(t)) ~ u2(t), - D(u2(t)) ~ 5 - 10 * sin(u1(t)) - 1.7 * u2(t)]; - -import ModelingToolkit: Interval -bcs = [u1(0) ~ -1, u2(0) ~ 7] -domains = [t ∈ Interval(0.0, 10.0)] -dt = 0.01 - -input_ = length(domains) # number of dimensions -n = 16 -chain = [Lux.Chain(Lux.Dense(input_, n, Lux.σ), Lux.Dense(n, n, Lux.σ), Lux.Dense(n, 1)) - for _ in 1:2] - -@named pde_system = PDESystem(eq, bcs, domains, [t], [u1(t), u2(t)]) - -strategy = NeuralPDE.GridTraining(dt) -discretization = PhysicsInformedNN(chain, strategy) -sym_prob = NeuralPDE.symbolic_discretize(pde_system, discretization) - -pde_loss_functions = sym_prob.loss_functions.pde_loss_functions -bc_loss_functions = sym_prob.loss_functions.bc_loss_functions - -callback = function (p, l) - println("loss: ", l) - # println("pde_losses: ", map(l_ -> l_(p), pde_loss_functions)) - # println("bcs_losses: ", map(l_ -> l_(p), bc_loss_functions)) - return false -end - -loss_functions = [pde_loss_functions; bc_loss_functions] - -function loss_function(θ, p) - sum(map(l -> l(θ), loss_functions)) -end - -f_ = OptimizationFunction(loss_function, Optimization.AutoZygote()) -prob = Optimization.OptimizationProblem(f_, sym_prob.flat_init_params) - -res = Optimization.solve(prob, - OptimizationOptimJL.BFGS(); - callback = callback, - maxiters = 1000) -phi = discretization.phi \ No newline at end of file From a5c3148724bf8d2569303ef912d589c985689e53 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sun, 21 Jan 2024 00:01:34 +0530 Subject: [PATCH 07/53] update advancedHMC_MCMC.jl --- src/advancedHMC_MCMC.jl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index e1ccc47261..6fee4a818e 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -467,17 +467,17 @@ Incase you are only solving the Equations for solution, do not provide dataset priors: pdf for W,b + pdf for ODE params """ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; - strategy = GridTraining, dataset = [nothing], - init_params = nothing, draw_samples = 1000, - physdt = 1 / 20.0, l2std = [0.05], - phystd = [0.05], priorsNNw = (0.0, 2.0), - param = [], nchains = 1, autodiff = false, - Kernel = HMC, - Adaptorkwargs = (Adaptor = StanHMCAdaptor, - Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), - Integratorkwargs = (Integrator = Leapfrog,), - MCMCkwargs = (n_leapfrog = 30,), - progress = false, verbose = false) + strategy = GridTraining, dataset = [nothing], + init_params = nothing, draw_samples = 1000, + physdt = 1 / 20.0, l2std = [0.05], + phystd = [0.05], priorsNNw = (0.0, 2.0), + param = [], nchains = 1, autodiff = false, + Kernel = HMC, + Adaptorkwargs = (Adaptor = StanHMCAdaptor, + Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), + Integratorkwargs = (Integrator = Leapfrog,), + MCMCkwargs = (n_leapfrog = 30,), + progress = false, verbose = false) # NN parameter prior mean and variance(PriorsNN must be a tuple) if isinplace(prob) From f8427a3aca489008be90cff14435bc87d755a8f4 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sun, 21 Jan 2024 00:07:36 +0530 Subject: [PATCH 08/53] update advancedHMC_MCMC.jl --- src/advancedHMC_MCMC.jl | 72 ++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index 6fee4a818e..1a2c47de0d 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -18,9 +18,9 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, init_params::I function LogTargetDensity(dim, prob, chain::Optimisers.Restructure, st, strategy, - dataset, - priors, phystd, l2std, autodiff, physdt, extraparams, - init_params::AbstractVector) + dataset, + priors, phystd, l2std, autodiff, physdt, extraparams, + init_params::AbstractVector) new{ typeof(chain), Nothing, @@ -42,9 +42,9 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, init_params) end function LogTargetDensity(dim, prob, chain::Lux.AbstractExplicitLayer, st, strategy, - dataset, - priors, phystd, l2std, autodiff, physdt, extraparams, - init_params::NamedTuple) + dataset, + priors, phystd, l2std, autodiff, physdt, extraparams, + init_params::NamedTuple) new{ typeof(chain), typeof(st), @@ -138,8 +138,8 @@ function physloglikelihood(Tar::LogTargetDensity, θ) end function getlogpdf(strategy::GridTraining, Tar::LogTargetDensity, f, autodiff::Bool, - tspan, - ode_params, θ) + tspan, + ode_params, θ) if Tar.dataset isa Vector{Nothing} t = collect(eltype(strategy.dx), tspan[1]:(strategy.dx):tspan[2]) else @@ -152,12 +152,12 @@ function getlogpdf(strategy::GridTraining, Tar::LogTargetDensity, f, autodiff::B end function getlogpdf(strategy::StochasticTraining, - Tar::LogTargetDensity, - f, - autodiff::Bool, - tspan, - ode_params, - θ) + Tar::LogTargetDensity, + f, + autodiff::Bool, + tspan, + ode_params, + θ) if Tar.dataset isa Vector{Nothing} t = [(tspan[2] - tspan[1]) * rand() + tspan[1] for i in 1:(strategy.points)] else @@ -170,9 +170,9 @@ function getlogpdf(strategy::StochasticTraining, end function getlogpdf(strategy::QuadratureTraining, Tar::LogTargetDensity, f, - autodiff::Bool, - tspan, - ode_params, θ) + autodiff::Bool, + tspan, + ode_params, θ) function integrand(t::Number, θ) innerdiff(Tar, f, autodiff, [t], θ, ode_params) end @@ -182,9 +182,9 @@ function getlogpdf(strategy::QuadratureTraining, Tar::LogTargetDensity, f, end function getlogpdf(strategy::WeightedIntervalTraining, Tar::LogTargetDensity, f, - autodiff::Bool, - tspan, - ode_params, θ) + autodiff::Bool, + tspan, + ode_params, θ) minT = tspan[1] maxT = tspan[2] @@ -217,7 +217,7 @@ end MvNormal likelihood at each `ti` in time `t` for ODE collocation residue with NN with parameters θ """ function innerdiff(Tar::LogTargetDensity, f, autodiff::Bool, t::AbstractVector, θ, - ode_params) + ode_params) # Tar used for phi and LogTargetDensity object attributes access out = Tar(t, θ[1:(length(θ) - Tar.extraparams)]) @@ -300,12 +300,12 @@ end nn OUTPUT AT t,θ ~ phi(t,θ) """ function (f::LogTargetDensity{C, S})(t::AbstractVector, - θ) where {C <: Optimisers.Restructure, S} + θ) where {C <: Optimisers.Restructure, S} f.prob.u0 .+ (t' .- f.prob.tspan[1]) .* f.chain(θ)(adapt(parameterless_type(θ), t')) end function (f::LogTargetDensity{C, S})(t::AbstractVector, - θ) where {C <: Lux.AbstractExplicitLayer, S} + θ) where {C <: Lux.AbstractExplicitLayer, S} θ = vector_to_parameters(θ, f.init_params) y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), t'), θ, f.st) ChainRulesCore.@ignore_derivatives f.st = st @@ -313,13 +313,13 @@ function (f::LogTargetDensity{C, S})(t::AbstractVector, end function (f::LogTargetDensity{C, S})(t::Number, - θ) where {C <: Optimisers.Restructure, S} + θ) where {C <: Optimisers.Restructure, S} # must handle paired odes hence u0 broadcasted f.prob.u0 .+ (t - f.prob.tspan[1]) * f.chain(θ)(adapt(parameterless_type(θ), [t])) end function (f::LogTargetDensity{C, S})(t::Number, - θ) where {C <: Lux.AbstractExplicitLayer, S} + θ) where {C <: Lux.AbstractExplicitLayer, S} θ = vector_to_parameters(θ, f.init_params) y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), [t]), θ, f.st) ChainRulesCore.@ignore_derivatives f.st = st @@ -467,17 +467,17 @@ Incase you are only solving the Equations for solution, do not provide dataset priors: pdf for W,b + pdf for ODE params """ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; - strategy = GridTraining, dataset = [nothing], - init_params = nothing, draw_samples = 1000, - physdt = 1 / 20.0, l2std = [0.05], - phystd = [0.05], priorsNNw = (0.0, 2.0), - param = [], nchains = 1, autodiff = false, - Kernel = HMC, - Adaptorkwargs = (Adaptor = StanHMCAdaptor, - Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), - Integratorkwargs = (Integrator = Leapfrog,), - MCMCkwargs = (n_leapfrog = 30,), - progress = false, verbose = false) + strategy = GridTraining, dataset = [nothing], + init_params = nothing, draw_samples = 1000, + physdt = 1 / 20.0, l2std = [0.05], + phystd = [0.05], priorsNNw = (0.0, 2.0), + param = [], nchains = 1, autodiff = false, + Kernel = HMC, + Adaptorkwargs = (Adaptor = StanHMCAdaptor, + Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), + Integratorkwargs = (Integrator = Leapfrog,), + MCMCkwargs = (n_leapfrog = 30,), + progress = false, verbose = false) # NN parameter prior mean and variance(PriorsNN must be a tuple) if isinplace(prob) From 237fb454ac763cd4853bf35a2fc38520643eae52 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sat, 3 Feb 2024 17:00:05 +0530 Subject: [PATCH 09/53] most of logic done --- Project.toml | 1 + src/NeuralPDE.jl | 1 + src/PDE_BPINN.jl | 177 +++++++++++++++++++++++++++------- test/BPINN_PDEinvsol_tests.jl | 163 +++++++++++++++++++++++++++++-- 4 files changed, 299 insertions(+), 43 deletions(-) diff --git a/Project.toml b/Project.toml index 1013977bad..d83eaa368d 100644 --- a/Project.toml +++ b/Project.toml @@ -10,6 +10,7 @@ ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" Cubature = "667455a9-e2ce-5579-9412-b964f529a492" +DataInterpolations = "82cc6244-b520-54b8-b5a6-8a565e85f1d0" DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" DiffEqNoiseProcess = "77a26b50-5914-5dd7-bc55-306e6241c503" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 931fb24a5c..80d9abdc62 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -24,6 +24,7 @@ using Symbolics: wrap, unwrap, arguments, operation using SymbolicUtils using AdvancedHMC, LogDensityProblems, LinearAlgebra, Functors, MCMCChains using MonteCarloMeasurements +using DataInterpolations: LinearInterpolation import ModelingToolkit: value, nameof, toexpr, build_expr, expand_derivatives import DomainSets: Domain, ClosedInterval diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index b63741e9b6..d5792d5e35 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -3,7 +3,7 @@ mutable struct PDELogTargetDensity{ D <: Union{Nothing, Vector{<:Matrix{<:Real}}}, P <: Vector{<:Distribution}, I, - F, + F, FF, PH, } dim::Int64 @@ -15,17 +15,18 @@ mutable struct PDELogTargetDensity{ extraparams::Int init_params::I full_loglikelihood::F + L2_loss2::FF Φ::PH function PDELogTargetDensity(dim, strategy, dataset, priors, allstd, names, extraparams, - init_params::AbstractVector, full_loglikelihood, Φ) + init_params::AbstractVector, full_loglikelihood, L2_loss2, Φ) new{ typeof(strategy), typeof(dataset), typeof(priors), typeof(init_params), - typeof(full_loglikelihood), + typeof(full_loglikelihood), typeof(L2_loss2), typeof(Φ), }(dim, strategy, @@ -35,19 +36,19 @@ mutable struct PDELogTargetDensity{ names, extraparams, init_params, - full_loglikelihood, + full_loglikelihood, L2_loss2, Φ) end function PDELogTargetDensity(dim, strategy, dataset, priors, allstd, names, extraparams, init_params::Union{NamedTuple, ComponentArrays.ComponentVector}, - full_loglikelihood, Φ) + full_loglikelihood, L2_loss2, Φ) new{ typeof(strategy), typeof(dataset), typeof(priors), typeof(init_params), - typeof(full_loglikelihood), + typeof(full_loglikelihood), typeof(L2_loss2), typeof(Φ), }(dim, strategy, @@ -57,23 +58,122 @@ mutable struct PDELogTargetDensity{ names, extraparams, init_params, - full_loglikelihood, + full_loglikelihood, L2_loss2, Φ) end end +# dataset_pde has normal matrix format +# dataset_bc has format of Vector{typeof(dataset_pde )} as each bc has different domain requirements +function get_symbols(dict_depvar_input, dataset, depvars) + # get datasets into splattable form + splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] + # splat datasets onto Linear interpolations tables + interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] + interps = Dict(depvars .=> interps) + + Dict_symbol_interps = Dict(depvar => (interps[depvar], dict_depvar_input[depvar]) + for depvar in depvars) + + tobe_subs = Dict() + for (a, b) in dict_depvar_input + tobe_subs[a] = eval(:($a($(b...)))) + end + + to_subs = Dict() + for (a, b) in Dict_symbol_interps + b1, b2 = b + to_subs[a] = eval(:($b1($(b2...)))) + end + return to_subs, tobe_subs +end + +function recur_expression(exp, Dict_differentials) + for in_exp in exp.args + if !(in_exp isa Expr) + # skip +,== symbols, characters etc + continue + + elseif in_exp.args[1] isa ModelingToolkit.Differential + # first symbol of differential term + # Dict_differentials for masking differential terms + # and resubstituting differentials in equations after putting in interpolations + Dict_differentials[eval(in_exp)] = Symbol("diff_$(length(Dict_differentials)+1)") + return + + else + recur_expression(in_exp, Dict_differentials) + end + end +end + +# get datafree loss functions for new loss type +# need to call merge_strategy_with_loss_function() variant after this +function merge_dataset_with_loss_function(pinnrep::NeuralPDE.PINNRepresentation, + dataset, + datafree_pde_loss_function, + datafree_bc_loss_function) + @unpack domains, eqs, bcs, dict_indvars, dict_depvars, flat_init_params = pinnrep + + eltypeθ = eltype(pinnrep.flat_init_params) + + train_sets = [[dataset[i][:, 2] for i in eachindex(dataset)], [[0;;], [0;;], [0;;]]] + + # the points in the domain and on the boundary + pde_train_sets, bcs_train_sets = train_sets + pde_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), + pde_train_sets) + bcs_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), + bcs_train_sets) + pde_loss_functions = [get_loss_function(_loss, _set, eltypeθ) + for (_loss, _set) in zip(datafree_pde_loss_function, + pde_train_sets)] + + bc_loss_functions = [get_loss_function(_loss, _set, eltypeθ) + for (_loss, _set) in zip(datafree_bc_loss_function, bcs_train_sets)] + + pde_loss_functions, bc_loss_functions +end + +function get_loss_function(loss_function, train_set, eltypeθ; τ = nothing) + loss = (θ) -> mean(abs2, loss_function(train_set, θ)) +end + +# for bc case, [bc]/bc eqs must be passed along with dataset_bc[i] +# and final loss for bc must be together in a vector(bcs has seperate type of dataset_bc) +# eqs is vector of pde eqs and dataset here is dataset_pde +# normally you get vector of losses +function get_loss_2(pinnrep, dataset, eqs) + depvars = pinnrep.depvars # order is same as dataset and interps + dict_depvar_input = pinnrep.dict_depvar_input + + to_subs, tobe_subs = get_symbols(dict_depvar_input, dataset, depvars) + interp_subs_dict = Dict(tobe_subs[depvar] => to_subs[depvar] for depvar in depvars) + + Dict_differentials = Dict() + exp = toexpr(eqs) + void_value = [recur_expression(exp_i, Dict_differentials) for exp_i in exp] + # Dict_differentials is now filled with Differential operator => diff_i key-value pairs + + # masking operation + a = substitute.(eqs, Ref(Dict_differentials)) + b = substitute.(a, Ref(interp_subs_dict)) + # reverse dict for re-substituing values of Differential(t)(u(t)) etc + rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) + eqs = substitute.(b, Ref(rev_Dict_differentials)) + # get losses + loss_functions = [NeuralPDE.build_loss_function(pinnrep, + eqs[i], + pinnrep.pde_indvars[i]) for i in eachindex(eqs)] +end + function LogDensityProblems.logdensity(Tar::PDELogTargetDensity, θ) # for parameter estimation neccesarry to use multioutput case return Tar.full_loglikelihood(setparameters(Tar, θ), - Tar.allstd) + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) - # + L2loss2(Tar, θ) + Tar.allstd) + priorlogpdf(Tar, θ) + Tar.L2_loss2(setparameters(Tar, θ), + Tar.allstd) end -# function L2loss2(Tar::PDELogTargetDensity, θ) -# return Tar.full_loglikelihood(setparameters(Tar, θ), -# Tar.allstd) -# end - function setparameters(Tar::PDELogTargetDensity, θ) names = Tar.names ps_new = θ[1:(end - Tar.extraparams)] @@ -131,6 +231,8 @@ function L2LossData(Tar::PDELogTargetDensity, θ) # dataset of form Vector[matrix_x, matrix_y, matrix_z] # matrix_i is of form [i,indvar1,indvar2,..] (needed in case if heterogenous domains) + # note that indvar1,indvar2.. cols can be different values for different depvar matrices + # order follows pinnrep.depvars orders of variables (order of declaration in @variables macro) # Phi is the trial solution for each NN in chain array # Creating logpdf( MvNormal(Phi(t,θ),std), dataset[i] ) @@ -188,27 +290,6 @@ function priorlogpdf(Tar::PDELogTargetDensity, θ) end end -function integratorchoice(Integratorkwargs, initial_ϵ) - Integrator = Integratorkwargs[:Integrator] - if Integrator == JitteredLeapfrog - jitter_rate = Integratorkwargs[:jitter_rate] - Integrator(initial_ϵ, jitter_rate) - elseif Integrator == TemperedLeapfrog - tempering_rate = Integratorkwargs[:tempering_rate] - Integrator(initial_ϵ, tempering_rate) - else - Integrator(initial_ϵ) - end -end - -function adaptorchoice(Adaptor, mma, ssa) - if Adaptor != AdvancedHMC.NoAdaptation() - Adaptor(mma, ssa) - else - AdvancedHMC.NoAdaptation() - end -end - function inference(samples, pinnrep, saveats, numensemble, ℓπ) domains = pinnrep.domains phi = pinnrep.phi @@ -353,6 +434,30 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; pinnrep = symbolic_discretize(pde_system, discretization) dataset_pde, dataset_bc = discretization.dataset + eqs = pinnrep.eqs + yuh1 = get_loss_2(pinnrep, dataset_pde, eqs) + eqs = pinnrep.bcs + yuh2 = get_loss_2(pinnrep, dataset_bc, eqs) + + pde_loss_functions, bc_loss_functions = merge_dataset_with_loss_function(pinnrep, + dataset, + yuh1, + yuh2) + + function L2_loss2(θ, allstd) + stdpdes, stdbcs, stdextra = allstd + pde_loglikelihoods = [logpdf(Normal(0, stdpdes[i]), pde_loss_function(θ)) + for (i, pde_loss_function) in enumerate(pde_loss_functions)] + + bc_loglikelihoods = [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) + for (j, bc_loss_function) in enumerate(bc_loss_functions)] + println("pde_loglikelihoods : ", pde_loglikelihoods) + println("bc_loglikelihoods : ", bc_loglikelihoods) + return sum(sum(pde_loglikelihoods) + sum(bc_loglikelihoods)) + end + + println(L2_loss2) + # WIP split dataset to respective equations if ((dataset_bc isa Nothing) && (dataset_pde isa Nothing)) dataset = nothing elseif dataset_bc isa Nothing @@ -441,7 +546,7 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; names, ninv, initial_nnθ, - full_weighted_loglikelihood, + full_weighted_loglikelihood, L2_loss2, Φ) Adaptor, Metric, targetacceptancerate = Adaptorkwargs[:Adaptor], diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 3521c8c913..4876328413 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -3,7 +3,7 @@ import ModelingToolkit: Interval, infimum, supremum using ForwardDiff, Distributions, OrdinaryDiffEq using Flux, AdvancedHMC, Statistics, Random, Functors using NeuralPDE, MonteCarloMeasurements -using ComponentArrays +using ComponentArrays, ModelingToolkit Random.seed!(100) @@ -32,9 +32,9 @@ initl, st = Lux.setup(Random.default_rng(), chainl) analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) timepoints = collect(0.0:(1 / 100.0):2.0) -u = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] -u = u .+ (u .* 0.2) .* randn(size(u)) -dataset = [hcat(u, timepoints)] +u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] +u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) +dataset = [hcat(u1, timepoints)] # plot(dataset[1][:, 2], dataset[1][:, 1]) # plot!(timepoints, u) @@ -51,7 +51,7 @@ ahmc_bayesian_pinn_pde(pde_system, phystd = [0.01], l2std = [0.01], priorsNNw = (0.0, 1.0), saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)]) + param = [LogNormal(6.0, 0.5)], progress = true) discretization = NeuralPDE.BayesianPINN([chainl], QuasiRandomTraining(200), @@ -175,7 +175,7 @@ discretization = NeuralPDE.BayesianPINN(chain, NeuralPDE.GridTraining([0.01]); sol1 = ahmc_bayesian_pinn_pde(pde_system, discretization; - draw_samples = 50, + draw_samples = 20, bcstd = [0.3, 0.3, 0.3], phystd = [0.1, 0.1, 0.1], l2std = [1, 1, 1], @@ -192,4 +192,153 @@ p_ = sol1.estimated_de_params[1] # plot!(sol1.timepoints[3]', pmean(sol1.ensemblesol[3])) @test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] -# @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] \ No newline at end of file +# @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] + +# # NEW LOSS FUNCTION CODE +# pinnrep = symbolic_discretize(pde_system, discretization) + +# # general equation with diff +# # now 1> substute u(t), phi(t) values from dataset and get multiple equations +# # phi[i] must be in numeric_derivative() form +# # derivative(phi, u, [x, y], εs, order, θ) - use parse_equations() and interp object to create loss function +# # this function must take interp objects(train sets) +# # dataset - get u(t), t from dataset interpolations object +# # make lhs-rhs loss +# # sum losses + +# using DataInterpolations + +# # dataset_pde has normal matrix format +# # dataset_bc has format of Vector{typeof(dataset_pde )} as each bc has different domain requirements +# function get_symbols(dict_depvar_input, dataset, depvars) +# # get datasets into splattable form +# splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] +# # splat datasets onto Linear interpolations tables +# interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] +# interps = Dict(depvars .=> interps) + +# Dict_symbol_interps = Dict(depvar => (interps[depvar], dict_depvar_input[depvar]) +# for depvar in depvars) + +# tobe_subs = Dict() +# for (a, b) in dict_depvar_input +# tobe_subs[a] = eval(:($a($(b...)))) +# end + +# to_subs = Dict() +# for (a, b) in Dict_symbol_interps +# b1, b2 = b +# to_subs[a] = eval(:($b1($(b2...)))) +# end +# return to_subs, tobe_subs +# end + +# function recur_expression(exp, Dict_differentials) +# for in_exp in exp.args +# if !(in_exp isa Expr) +# # skip +,== symbols, characters etc +# continue + +# elseif in_exp.args[1] isa ModelingToolkit.Differential +# # first symbol of differential term +# # Dict_differentials for masking differential terms +# # and resubstituting differentials in equations after putting in interpolations +# Dict_differentials[eval(in_exp)] = Symbol("diff_$(length(Dict_differentials)+1)") +# return + +# else +# recur_expression(in_exp, Dict_differentials) +# end +# end +# end + +# # get datafree loss functions for new loss type +# # need to call merge_strategy_with_loss_function() variant after this +# function merge_dataset_with_loss_function(pinnrep::NeuralPDE.PINNRepresentation, +# dataset, +# datafree_pde_loss_function, +# datafree_bc_loss_function) +# @unpack domains, eqs, bcs, dict_indvars, dict_depvars, flat_init_params = pinnrep + +# eltypeθ = eltype(pinnrep.flat_init_params) + +# train_sets = [[dataset[i][:, 2] for i in eachindex(dataset)], [[0;;], [0;;], [0;;]]] + +# # the points in the domain and on the boundary +# pde_train_sets, bcs_train_sets = train_sets +# # pde_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), +# # pde_train_sets) +# # bcs_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), +# # bcs_train_sets) +# pde_loss_functions = [get_loss_function(_loss, _set, eltypeθ) +# for (_loss, _set) in zip(datafree_pde_loss_function, +# pde_train_sets)] + +# bc_loss_functions = [get_loss_function(_loss, _set, eltypeθ) +# for (_loss, _set) in zip(datafree_bc_loss_function, bcs_train_sets)] + +# pde_loss_functions, bc_loss_functions +# end + +# function get_loss_function(loss_function, train_set, eltypeθ; τ = nothing) +# loss = (θ) -> mean(abs2, loss_function(train_set, θ)) +# end + +# # for bc case, [bc]/bc eqs must be passed along with dataset_bc[i] +# # and final loss for bc must be together in a vector(bcs has seperate type of dataset_bc) +# # eqs is vector of pde eqs and dataset here is dataset_pde +# # normally you get vector of losses +# function get_loss_2(pinnrep, dataset, eqs) +# depvars = pinnrep.depvars # order is same as dataset and interps +# dict_depvar_input = pinnrep.dict_depvar_input + +# to_subs, tobe_subs = get_symbols(dict_depvar_input, dataset, depvars) +# interp_subs_dict = Dict(tobe_subs[depvar] => to_subs[depvar] for depvar in depvars) + +# Dict_differentials = Dict() +# exp = toexpr(eqs) +# void_value = [recur_expression(exp_i, Dict_differentials) for exp_i in exp] +# # Dict_differentials is now filled with Differential operator => diff_i key-value pairs + +# # masking operation +# a = substitute.(eqs, Ref(Dict_differentials)) +# b = substitute.(a, Ref(interp_subs_dict)) +# # reverse dict for re-substituing values of Differential(t)(u(t)) etc +# rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) +# eqs = substitute.(b, Ref(rev_Dict_differentials)) +# # get losses +# loss_functions = [NeuralPDE.build_loss_function(pinnrep, +# eqs[i], +# pinnrep.pde_indvars[i]) for i in eachindex(eqs)] +# end + +# eqs = pde_system.eqs +# yuh1 = get_loss_2(pinnrep, dataset, eqs) +# eqs = pinnrep.bcs +# yuh2 = get_loss_2(pinnrep, dataset, eqs) + +# pde_loss_functions, bc_loss_functions = merge_dataset_with_loss_function(pinnrep, +# dataset, +# yuh1, +# yuh2) + +# pde_loss_functions() +# # logic for recursion formula to parse differentials +# # # this below has the whole differential term +# # toexpr(pde_system.eqs[1]).args[2].args[3].args[3] isa ModelingToolkit.Differential +# # toexpr(pde_system.eqs[1]).args[2].args[3].args[3] +# # # .args[1] isa ModelingToolkit.Differential + +# # logic for interpolation and indvars splatting to get Equation parsing terms +# # splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] +# # # splat datasets onto Linear interpolations tables +# # interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] +# # interps = Dict(depvars .=> interps) +# # get datasets into splattable form +# # splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] +# # # splat datasets onto Linear interpolations tables +# # yu = [LinearInterpolation(splat_i...) for splat_i in splat_form] +# # Symbol(:($(yu[1]))) + +# # logic to contrauct dict to feed for masking +# # Dict(interps[depvar] => dict_depvar_input[depvar] for depvar in depvars) \ No newline at end of file From f1e031559f5426ce7a75453369d44ac13491a3d3 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sun, 4 Feb 2024 01:43:13 +0530 Subject: [PATCH 10/53] removed duplicate methods --- src/PDE_BPINN.jl | 21 +++++++++++++++++++++ src/advancedHMC_MCMC.jl | 21 --------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index d5792d5e35..2e0401d843 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -376,6 +376,27 @@ function inference(samples, pinnrep, saveats, numensemble, ℓπ) end end +function integratorchoice(Integratorkwargs, initial_ϵ) + Integrator = Integratorkwargs[:Integrator] + if Integrator == JitteredLeapfrog + jitter_rate = Integratorkwargs[:jitter_rate] + Integrator(initial_ϵ, jitter_rate) + elseif Integrator == TemperedLeapfrog + tempering_rate = Integratorkwargs[:tempering_rate] + Integrator(initial_ϵ, tempering_rate) + else + Integrator(initial_ϵ) + end +end + +function adaptorchoice(Adaptor, mma, ssa) + if Adaptor != AdvancedHMC.NoAdaptation() + Adaptor(mma, ssa) + else + AdvancedHMC.NoAdaptation() + end +end + """ ```julia ahmc_bayesian_pinn_pde(pde_system, discretization; diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index 1a2c47de0d..1efbadd4bd 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -351,27 +351,6 @@ function kernelchoice(Kernel, MCMCkwargs) end end -function integratorchoice(Integratorkwargs, initial_ϵ) - Integrator = Integratorkwargs[:Integrator] - if Integrator == JitteredLeapfrog - jitter_rate = Integratorkwargs[:jitter_rate] - Integrator(initial_ϵ, jitter_rate) - elseif Integrator == TemperedLeapfrog - tempering_rate = Integratorkwargs[:tempering_rate] - Integrator(initial_ϵ, tempering_rate) - else - Integrator(initial_ϵ) - end -end - -function adaptorchoice(Adaptor, mma, ssa) - if Adaptor != AdvancedHMC.NoAdaptation() - Adaptor(mma, ssa) - else - AdvancedHMC.NoAdaptation() - end -end - """ ```julia ahmc_bayesian_pinn_ode(prob, chain; strategy = GridTraining, From 4c88dd4ec18778f25caba8c6053c6f62d4f9223c Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sun, 4 Feb 2024 02:02:02 +0530 Subject: [PATCH 11/53] update BPINN_PDE_tests.jl --- test/BPINN_PDE_tests.jl | 151 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/test/BPINN_PDE_tests.jl b/test/BPINN_PDE_tests.jl index 6dd3637f5a..7a5e47b83d 100644 --- a/test/BPINN_PDE_tests.jl +++ b/test/BPINN_PDE_tests.jl @@ -206,3 +206,154 @@ end u_predict = pmean(sol1.ensemblesol[1]) @test u_predict≈u_real atol=0.8 end + + + +# # NEW LOSS FUNCTION CODE +pinnrep = symbolic_discretize(pde_system, discretization) + +# general equation with diff +# now 1> substute u(t), phi(t) values from dataset and get multiple equations +# phi[i] must be in numeric_derivative() form +# derivative(phi, u, [x, y], εs, order, θ) - use parse_equations() and interp object to create loss function +# this function must take interp objects(train sets) +# dataset - get u(t), t from dataset interpolations object +# make lhs-rhs loss +# sum losses + +using DataInterpolations + +# dataset_pde has normal matrix format +# dataset_bc has format of Vector{typeof(dataset_pde )} as each bc has different domain requirements +function get_symbols(dict_depvar_input, dataset, depvars) + # get datasets into splattable form + splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] + # splat datasets onto Linear interpolations tables + interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] + interps = Dict(depvars .=> interps) + + Dict_symbol_interps = Dict(depvar => (interps[depvar], dict_depvar_input[depvar]) + for depvar in depvars) + + tobe_subs = Dict() + for (a, b) in dict_depvar_input + tobe_subs[a] = eval(:($a($(b...)))) + end + + to_subs = Dict() + for (a, b) in Dict_symbol_interps + b1, b2 = b + to_subs[a] = eval(:($b1($(b2...)))) + end + return to_subs, tobe_subs +end + +function recur_expression(exp, Dict_differentials) + for in_exp in exp.args + if !(in_exp isa Expr) + # skip +,== symbols, characters etc + continue + + elseif in_exp.args[1] isa ModelingToolkit.Differential + # first symbol of differential term + # Dict_differentials for masking differential terms + # and resubstituting differentials in equations after putting in interpolations + Dict_differentials[eval(in_exp)] = Symbol("diff_$(length(Dict_differentials)+1)") + return + + else + recur_expression(in_exp, Dict_differentials) + end + end +end + +# get datafree loss functions for new loss type +# need to call merge_strategy_with_loss_function() variant after this +function merge_dataset_with_loss_function(pinnrep::NeuralPDE.PINNRepresentation, + dataset, + datafree_pde_loss_function, + datafree_bc_loss_function) + @unpack domains, eqs, bcs, dict_indvars, dict_depvars, flat_init_params = pinnrep + + eltypeθ = eltype(pinnrep.flat_init_params) + + train_sets = [[dataset[i][:, 2] for i in eachindex(dataset)], [[0;;], [0;;], [0;;]]] + + # the points in the domain and on the boundary + pde_train_sets, bcs_train_sets = train_sets + # pde_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), + # pde_train_sets) + # bcs_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), + # bcs_train_sets) + pde_loss_functions = [get_loss_function(_loss, _set, eltypeθ) + for (_loss, _set) in zip(datafree_pde_loss_function, + pde_train_sets)] + + bc_loss_functions = [get_loss_function(_loss, _set, eltypeθ) + for (_loss, _set) in zip(datafree_bc_loss_function, bcs_train_sets)] + + pde_loss_functions, bc_loss_functions +end + +function get_loss_function(loss_function, train_set, eltypeθ; τ = nothing) + loss = (θ) -> mean(abs2, loss_function(train_set, θ)) +end + +# for bc case, [bc]/bc eqs must be passed along with dataset_bc[i] +# and final loss for bc must be together in a vector(bcs has seperate type of dataset_bc) +# eqs is vector of pde eqs and dataset here is dataset_pde +# normally you get vector of losses +function get_loss_2(pinnrep, dataset, eqs) + depvars = pinnrep.depvars # order is same as dataset and interps + dict_depvar_input = pinnrep.dict_depvar_input + + to_subs, tobe_subs = get_symbols(dict_depvar_input, dataset, depvars) + interp_subs_dict = Dict(tobe_subs[depvar] => to_subs[depvar] for depvar in depvars) + + Dict_differentials = Dict() + exp = toexpr(eqs) + void_value = [recur_expression(exp_i, Dict_differentials) for exp_i in exp] + # Dict_differentials is now filled with Differential operator => diff_i key-value pairs + + # masking operation + a = substitute.(eqs, Ref(Dict_differentials)) + b = substitute.(a, Ref(interp_subs_dict)) + # reverse dict for re-substituing values of Differential(t)(u(t)) etc + rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) + eqs = substitute.(b, Ref(rev_Dict_differentials)) + # get losses + loss_functions = [NeuralPDE.build_loss_function(pinnrep, + eqs[i], + pinnrep.pde_indvars[i]) for i in eachindex(eqs)] +end + +eqs = pde_system.eqs +yuh1 = get_loss_2(pinnrep, dataset, eqs) +eqs = pinnrep.bcs +yuh2 = get_loss_2(pinnrep, dataset, eqs) + +pde_loss_functions, bc_loss_functions = merge_dataset_with_loss_function(pinnrep, + dataset, + yuh1, + yuh2) + +pde_loss_functions() +# logic for recursion formula to parse differentials +# # this below has the whole differential term +# toexpr(pde_system.eqs[1]).args[2].args[3].args[3] isa ModelingToolkit.Differential +# toexpr(pde_system.eqs[1]).args[2].args[3].args[3] +# # .args[1] isa ModelingToolkit.Differential + +# logic for interpolation and indvars splatting to get Equation parsing terms +# splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] +# # splat datasets onto Linear interpolations tables +# interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] +# interps = Dict(depvars .=> interps) +# get datasets into splattable form +# splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] +# # splat datasets onto Linear interpolations tables +# yu = [LinearInterpolation(splat_i...) for splat_i in splat_form] +# Symbol(:($(yu[1]))) + +# logic to contrauct dict to feed for masking +# Dict(interps[depvar] => dict_depvar_input[depvar] for depvar in depvars) \ No newline at end of file From f7836fdde4f57fbeb18ebe46dc89f4995dbbb2b0 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sun, 4 Feb 2024 02:03:46 +0530 Subject: [PATCH 12/53] update BPINN_PDE_tests.jl --- test/BPINN_PDE_tests.jl | 153 +--------------------------------------- 1 file changed, 1 insertion(+), 152 deletions(-) diff --git a/test/BPINN_PDE_tests.jl b/test/BPINN_PDE_tests.jl index 7a5e47b83d..cd8f6ef466 100644 --- a/test/BPINN_PDE_tests.jl +++ b/test/BPINN_PDE_tests.jl @@ -205,155 +205,4 @@ end u_real = vec([analytic_sol_func(t) for t in ts]) u_predict = pmean(sol1.ensemblesol[1]) @test u_predict≈u_real atol=0.8 -end - - - -# # NEW LOSS FUNCTION CODE -pinnrep = symbolic_discretize(pde_system, discretization) - -# general equation with diff -# now 1> substute u(t), phi(t) values from dataset and get multiple equations -# phi[i] must be in numeric_derivative() form -# derivative(phi, u, [x, y], εs, order, θ) - use parse_equations() and interp object to create loss function -# this function must take interp objects(train sets) -# dataset - get u(t), t from dataset interpolations object -# make lhs-rhs loss -# sum losses - -using DataInterpolations - -# dataset_pde has normal matrix format -# dataset_bc has format of Vector{typeof(dataset_pde )} as each bc has different domain requirements -function get_symbols(dict_depvar_input, dataset, depvars) - # get datasets into splattable form - splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] - # splat datasets onto Linear interpolations tables - interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] - interps = Dict(depvars .=> interps) - - Dict_symbol_interps = Dict(depvar => (interps[depvar], dict_depvar_input[depvar]) - for depvar in depvars) - - tobe_subs = Dict() - for (a, b) in dict_depvar_input - tobe_subs[a] = eval(:($a($(b...)))) - end - - to_subs = Dict() - for (a, b) in Dict_symbol_interps - b1, b2 = b - to_subs[a] = eval(:($b1($(b2...)))) - end - return to_subs, tobe_subs -end - -function recur_expression(exp, Dict_differentials) - for in_exp in exp.args - if !(in_exp isa Expr) - # skip +,== symbols, characters etc - continue - - elseif in_exp.args[1] isa ModelingToolkit.Differential - # first symbol of differential term - # Dict_differentials for masking differential terms - # and resubstituting differentials in equations after putting in interpolations - Dict_differentials[eval(in_exp)] = Symbol("diff_$(length(Dict_differentials)+1)") - return - - else - recur_expression(in_exp, Dict_differentials) - end - end -end - -# get datafree loss functions for new loss type -# need to call merge_strategy_with_loss_function() variant after this -function merge_dataset_with_loss_function(pinnrep::NeuralPDE.PINNRepresentation, - dataset, - datafree_pde_loss_function, - datafree_bc_loss_function) - @unpack domains, eqs, bcs, dict_indvars, dict_depvars, flat_init_params = pinnrep - - eltypeθ = eltype(pinnrep.flat_init_params) - - train_sets = [[dataset[i][:, 2] for i in eachindex(dataset)], [[0;;], [0;;], [0;;]]] - - # the points in the domain and on the boundary - pde_train_sets, bcs_train_sets = train_sets - # pde_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), - # pde_train_sets) - # bcs_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), - # bcs_train_sets) - pde_loss_functions = [get_loss_function(_loss, _set, eltypeθ) - for (_loss, _set) in zip(datafree_pde_loss_function, - pde_train_sets)] - - bc_loss_functions = [get_loss_function(_loss, _set, eltypeθ) - for (_loss, _set) in zip(datafree_bc_loss_function, bcs_train_sets)] - - pde_loss_functions, bc_loss_functions -end - -function get_loss_function(loss_function, train_set, eltypeθ; τ = nothing) - loss = (θ) -> mean(abs2, loss_function(train_set, θ)) -end - -# for bc case, [bc]/bc eqs must be passed along with dataset_bc[i] -# and final loss for bc must be together in a vector(bcs has seperate type of dataset_bc) -# eqs is vector of pde eqs and dataset here is dataset_pde -# normally you get vector of losses -function get_loss_2(pinnrep, dataset, eqs) - depvars = pinnrep.depvars # order is same as dataset and interps - dict_depvar_input = pinnrep.dict_depvar_input - - to_subs, tobe_subs = get_symbols(dict_depvar_input, dataset, depvars) - interp_subs_dict = Dict(tobe_subs[depvar] => to_subs[depvar] for depvar in depvars) - - Dict_differentials = Dict() - exp = toexpr(eqs) - void_value = [recur_expression(exp_i, Dict_differentials) for exp_i in exp] - # Dict_differentials is now filled with Differential operator => diff_i key-value pairs - - # masking operation - a = substitute.(eqs, Ref(Dict_differentials)) - b = substitute.(a, Ref(interp_subs_dict)) - # reverse dict for re-substituing values of Differential(t)(u(t)) etc - rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) - eqs = substitute.(b, Ref(rev_Dict_differentials)) - # get losses - loss_functions = [NeuralPDE.build_loss_function(pinnrep, - eqs[i], - pinnrep.pde_indvars[i]) for i in eachindex(eqs)] -end - -eqs = pde_system.eqs -yuh1 = get_loss_2(pinnrep, dataset, eqs) -eqs = pinnrep.bcs -yuh2 = get_loss_2(pinnrep, dataset, eqs) - -pde_loss_functions, bc_loss_functions = merge_dataset_with_loss_function(pinnrep, - dataset, - yuh1, - yuh2) - -pde_loss_functions() -# logic for recursion formula to parse differentials -# # this below has the whole differential term -# toexpr(pde_system.eqs[1]).args[2].args[3].args[3] isa ModelingToolkit.Differential -# toexpr(pde_system.eqs[1]).args[2].args[3].args[3] -# # .args[1] isa ModelingToolkit.Differential - -# logic for interpolation and indvars splatting to get Equation parsing terms -# splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] -# # splat datasets onto Linear interpolations tables -# interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] -# interps = Dict(depvars .=> interps) -# get datasets into splattable form -# splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] -# # splat datasets onto Linear interpolations tables -# yu = [LinearInterpolation(splat_i...) for splat_i in splat_form] -# Symbol(:($(yu[1]))) - -# logic to contrauct dict to feed for masking -# Dict(interps[depvar] => dict_depvar_input[depvar] for depvar in depvars) \ No newline at end of file +end \ No newline at end of file From a3a0cb547dba13ec4c4f5f5f2a2a86246809bfde Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sun, 4 Feb 2024 02:24:06 +0530 Subject: [PATCH 13/53] keeping bayesian directory files in sync with master --- src/{bayesian => }/BPINN_ode.jl | 0 src/NeuralPDE.jl | 6 +++--- src/{bayesian => }/advancedHMC_MCMC.jl | 22 ---------------------- src/{bayesian => }/collocated_estim.jl | 0 4 files changed, 3 insertions(+), 25 deletions(-) rename src/{bayesian => }/BPINN_ode.jl (100%) rename src/{bayesian => }/advancedHMC_MCMC.jl (96%) rename src/{bayesian => }/collocated_estim.jl (100%) diff --git a/src/bayesian/BPINN_ode.jl b/src/BPINN_ode.jl similarity index 100% rename from src/bayesian/BPINN_ode.jl rename to src/BPINN_ode.jl diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 66d6e9c58f..57d13d255e 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -50,10 +50,10 @@ include("dae_solve.jl") include("transform_inf_integral.jl") include("discretize.jl") include("neural_adapter.jl") -include("bayesian/advancedHMC_MCMC.jl") -include("bayesian/BPINN_ode.jl") +include("advancedHMC_MCMC.jl") +include("BPINN_ode.jl") include("PDE_BPINN.jl") -include("bayesian/collocated_estim.jl") +include("collocated_estim.jl") export NNODE, TerminalPDEProblem, NNPDEHan, NNPDENS, NNRODE, NNDAE, KolmogorovPDEProblem, NNKolmogorov, NNStopping, ParamKolmogorovPDEProblem, diff --git a/src/bayesian/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl similarity index 96% rename from src/bayesian/advancedHMC_MCMC.jl rename to src/advancedHMC_MCMC.jl index 5e995ebfdb..ada4539b10 100644 --- a/src/bayesian/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -290,25 +290,9 @@ function generate_Tar(chain::Lux.AbstractExplicitLayer, init_params::Nothing) return θ, chain, st end -function generate_Tar(chain::Flux.Chain, init_params) - θ, re = Flux.destructure(chain) - return init_params, re, nothing -end - -function generate_Tar(chain::Flux.Chain, init_params::Nothing) - θ, re = Flux.destructure(chain) - # find_good_stepsize,phasepoint takes only float64 - return θ, re, nothing -end - """ nn OUTPUT AT t,θ ~ phi(t,θ) """ -function (f::LogTargetDensity{C, S})(t::AbstractVector, - θ) where {C <: Optimisers.Restructure, S} - f.prob.u0 .+ (t' .- f.prob.tspan[1]) .* f.chain(θ)(adapt(parameterless_type(θ), t')) -end - function (f::LogTargetDensity{C, S})(t::AbstractVector, θ) where {C <: Lux.AbstractExplicitLayer, S} θ = vector_to_parameters(θ, f.init_params) @@ -317,12 +301,6 @@ function (f::LogTargetDensity{C, S})(t::AbstractVector, f.prob.u0 .+ (t' .- f.prob.tspan[1]) .* y end -function (f::LogTargetDensity{C, S})(t::Number, - θ) where {C <: Optimisers.Restructure, S} - # must handle paired odes hence u0 broadcasted - f.prob.u0 .+ (t - f.prob.tspan[1]) * f.chain(θ)(adapt(parameterless_type(θ), [t])) -end - function (f::LogTargetDensity{C, S})(t::Number, θ) where {C <: Lux.AbstractExplicitLayer, S} θ = vector_to_parameters(θ, f.init_params) diff --git a/src/bayesian/collocated_estim.jl b/src/collocated_estim.jl similarity index 100% rename from src/bayesian/collocated_estim.jl rename to src/collocated_estim.jl From e0803b15e5e16b753032d0f9e7908a2da8fec0bb Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sun, 4 Feb 2024 02:45:03 +0530 Subject: [PATCH 14/53] changes to sync with master --- src/advancedHMC_MCMC.jl | 188 +++++++++++++++++++--------------------- 1 file changed, 87 insertions(+), 101 deletions(-) diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index ada4539b10..e582be4f64 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -68,9 +68,11 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, end """ -cool function to convert parameter's vector to ComponentArray of parameters (for Lux Chain: vector of samples -> Lux ComponentArrays) +Function needed for converting vector of sampled parameters into ComponentVector in case of Lux chain output, derivatives +the sampled parameters are of exotic type `Dual` due to ForwardDiff's autodiff tagging. """ -function vector_to_parameters(ps_new::AbstractVector, ps::NamedTuple) +function vector_to_parameters(ps_new::AbstractVector, + ps::Union{NamedTuple, ComponentArrays.ComponentVector}) @assert length(ps_new) == Lux.parameterlength(ps) i = 1 function get_ps(x) @@ -81,6 +83,8 @@ function vector_to_parameters(ps_new::AbstractVector, ps::NamedTuple) return Functors.fmap(get_ps, ps) end +vector_to_parameters(ps_new::AbstractVector, ps::AbstractVector) = ps_new + function LogDensityProblems.logdensity(Tar::LogTargetDensity, θ) if Tar.estim_collocate return physloglikelihood(Tar, θ)/length(Tar.dataset[1]) + priorweights(Tar, θ) + L2LossData(Tar, θ)/length(Tar.dataset[1]) + L2loss2(Tar, θ)/length(Tar.dataset[1]) @@ -96,7 +100,7 @@ function LogDensityProblems.capabilities(::LogTargetDensity) end """ -L2 loss loglikelihood(needed for ODE parameter estimation) +L2 loss loglikelihood(needed for ODE parameter estimation). """ function L2LossData(Tar::LogTargetDensity, θ) # check if dataset is provided @@ -110,9 +114,8 @@ function L2LossData(Tar::LogTargetDensity, θ) for i in 1:length(Tar.prob.u0) # for u[i] ith vector must be added to dataset,nn[1,:] is the dx in lotka_volterra L2logprob += logpdf(MvNormal(nn[i, :], - LinearAlgebra.Diagonal(map(abs2, - Tar.l2std[i] .* - ones(length(Tar.dataset[i]))))), + LinearAlgebra.Diagonal(abs2.(Tar.l2std[i] .* + ones(length(Tar.dataset[i]))))), Tar.dataset[i]) end return L2logprob @@ -120,7 +123,7 @@ function L2LossData(Tar::LogTargetDensity, θ) end """ -physics loglikelihood over problem timespan + dataset timepoints +Physics loglikelihood over problem timespan + dataset timepoints. """ function physloglikelihood(Tar::LogTargetDensity, θ) f = Tar.prob.f @@ -218,7 +221,7 @@ function getlogpdf(strategy::WeightedIntervalTraining, Tar::LogTargetDensity, f, end """ -MvNormal likelihood at each `ti` in time `t` for ODE collocation residue with NN with parameters θ +MvNormal likelihood at each `ti` in time `t` for ODE collocation residue with NN with parameters θ. """ function innerdiff(Tar::LogTargetDensity, f, autodiff::Bool, t::AbstractVector, θ, ode_params) @@ -251,14 +254,13 @@ function innerdiff(Tar::LogTargetDensity, f, autodiff::Bool, t::AbstractVector, # N dimensional vector if N outputs for NN(each row has logpdf of i[i] where u is vector of dependant variables) return [logpdf(MvNormal(vals[i, :], - LinearAlgebra.Diagonal(map(abs2, - Tar.phystd[i] .* - ones(length(vals[i, :]))))), + LinearAlgebra.Diagonal(abs2.(Tar.phystd[i] .* + ones(length(vals[i, :]))))), zeros(length(vals[i, :]))) for i in 1:length(Tar.prob.u0)] end """ -prior logpdf for NN parameters + ODE constants +Prior logpdf for NN parameters + ODE constants. """ function priorweights(Tar::LogTargetDensity, θ) allparams = Tar.priors @@ -291,10 +293,10 @@ function generate_Tar(chain::Lux.AbstractExplicitLayer, init_params::Nothing) end """ -nn OUTPUT AT t,θ ~ phi(t,θ) +NN OUTPUT AT t,θ ~ phi(t,θ). """ function (f::LogTargetDensity{C, S})(t::AbstractVector, - θ) where {C <: Lux.AbstractExplicitLayer, S} + θ) where {C <: Lux.AbstractExplicitLayer, S} θ = vector_to_parameters(θ, f.init_params) y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), t'), θ, f.st) ChainRulesCore.@ignore_derivatives f.st = st @@ -302,7 +304,7 @@ function (f::LogTargetDensity{C, S})(t::AbstractVector, end function (f::LogTargetDensity{C, S})(t::Number, - θ) where {C <: Lux.AbstractExplicitLayer, S} + θ) where {C <: Lux.AbstractExplicitLayer, S} θ = vector_to_parameters(θ, f.init_params) y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), [t]), θ, f.st) ChainRulesCore.@ignore_derivatives f.st = st @@ -310,7 +312,7 @@ function (f::LogTargetDensity{C, S})(t::Number, end """ -similar to ode_dfdx() in NNODE/ode_solve.jl +Similar to ode_dfdx() in NNODE. """ function NNodederi(phi::LogTargetDensity, t::AbstractVector, θ, autodiff::Bool) if autodiff @@ -334,40 +336,19 @@ function kernelchoice(Kernel, MCMCkwargs) end end -function integratorchoice(Integratorkwargs, initial_ϵ) - Integrator = Integratorkwargs[:Integrator] - if Integrator == JitteredLeapfrog - jitter_rate = Integratorkwargs[:jitter_rate] - Integrator(initial_ϵ, jitter_rate) - elseif Integrator == TemperedLeapfrog - tempering_rate = Integratorkwargs[:tempering_rate] - Integrator(initial_ϵ, tempering_rate) - else - Integrator(initial_ϵ) - end -end - -function adaptorchoice(Adaptor, mma, ssa) - if Adaptor != AdvancedHMC.NoAdaptation() - Adaptor(mma, ssa) - else - AdvancedHMC.NoAdaptation() - end -end - """ -```julia -ahmc_bayesian_pinn_ode(prob, chain; strategy = GridTraining, - dataset = [nothing],init_params = nothing, - draw_samples = 1000, physdt = 1 / 20.0f0,l2std = [0.05], - phystd = [0.05], priorsNNw = (0.0, 2.0), - param = [], nchains = 1, autodiff = false, Kernel = HMC, - Adaptorkwargs = (Adaptor = StanHMCAdaptor, - Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), - Integratorkwargs = (Integrator = Leapfrog,), - MCMCkwargs = (n_leapfrog = 30,), - progress = false, verbose = false) -``` + ahmc_bayesian_pinn_ode(prob, chain; strategy = GridTraining, + dataset = [nothing],init_params = nothing, + draw_samples = 1000, physdt = 1 / 20.0f0,l2std = [0.05], + phystd = [0.05], priorsNNw = (0.0, 2.0), + param = [], nchains = 1, autodiff = false, Kernel = HMC, + Adaptorkwargs = (Adaptor = StanHMCAdaptor, + Metric = DiagEuclideanMetric, + targetacceptancerate = 0.8), + Integratorkwargs = (Integrator = Leapfrog,), + MCMCkwargs = (n_leapfrog = 30,), + progress = false, verbose = false) + !!! warn Note that ahmc_bayesian_pinn_ode() only supports ODEs which are written in the out-of-place form, i.e. @@ -375,85 +356,82 @@ ahmc_bayesian_pinn_ode(prob, chain; strategy = GridTraining, will exit with an error. ## Example + +```julia linear = (u, p, t) -> -u / p[1] + exp(t / p[2]) * cos(t) tspan = (0.0, 10.0) u0 = 0.0 p = [5.0, -5.0] prob = ODEProblem(linear, u0, tspan, p) -# CREATE DATASET (Necessity for accurate Parameter estimation) +### CREATE DATASET (Necessity for accurate Parameter estimation) sol = solve(prob, Tsit5(); saveat = 0.05) u = sol.u[1:100] time = sol.t[1:100] -# dataset and BPINN create +### dataset and BPINN create x̂ = collect(Float64, Array(u) + 0.05 * randn(size(u))) dataset = [x̂, time] -chainflux1 = Flux.Chain(Flux.Dense(1, 5, tanh), Flux.Dense(5, 5, tanh), Flux.Dense(5, 1) - -# simply solving ode here hence better to not pass dataset(uses ode params specified in prob) -fh_mcmc_chainflux1, fhsamplesflux1, fhstatsflux1 = ahmc_bayesian_pinn_ode(prob,chainflux1, - dataset = dataset, - draw_samples = 1500, - l2std = [0.05], - phystd = [0.05], - priorsNNw = (0.0,3.0)) - -# solving ode + estimating parameters hence dataset needed to optimize parameters upon + Pior Distributions for ODE params -fh_mcmc_chainflux2, fhsamplesflux2, fhstatsflux2 = ahmc_bayesian_pinn_ode(prob,chainflux1, - dataset = dataset, - draw_samples = 1500, - l2std = [0.05], - phystd = [0.05], - priorsNNw = (0.0,3.0), - param = [Normal(6.5,0.5),Normal(-3,0.5)]) - -## NOTES +chain1 = Lux.Chain(Lux.Dense(1, 5, tanh), Lux.Dense(5, 5, tanh), Lux.Dense(5, 1) + +### simply solving ode here hence better to not pass dataset(uses ode params specified in prob) +fh_mcmc_chain1, fhsamples1, fhstats1 = ahmc_bayesian_pinn_ode(prob, chain1, + dataset = dataset, + draw_samples = 1500, + l2std = [0.05], + phystd = [0.05], + priorsNNw = (0.0,3.0)) + +### solving ode + estimating parameters hence dataset needed to optimize parameters upon + Pior Distributions for ODE params +fh_mcmc_chain2, fhsamples2, fhstats2 = ahmc_bayesian_pinn_ode(prob, chain1, + dataset = dataset, + draw_samples = 1500, + l2std = [0.05], + phystd = [0.05], + priorsNNw = (0.0,3.0), + param = [Normal(6.5,0.5), Normal(-3,0.5)]) +``` + +## NOTES + Dataset is required for accurate Parameter estimation + solving equations Incase you are only solving the Equations for solution, do not provide dataset ## Positional Arguments -* `prob`: DEProblem(out of place and the function signature should be f(u,p,t) -* `chain`: Lux/Flux Neural Netork which would be made the Bayesian PINN + +* `prob`: DEProblem(out of place and the function signature should be f(u,p,t). +* `chain`: Lux Neural Netork which would be made the Bayesian PINN. ## Keyword Arguments + * `strategy`: The training strategy used to choose the points for the evaluations. By default GridTraining is used with given physdt discretization. -* `dataset`: Vector containing Vectors of corresponding u,t values * `init_params`: intial parameter values for BPINN (ideally for multiple chains different initializations preferred) -* `nchains`: number of chains you want to sample (random initialisation of params by default) +* `nchains`: number of chains you want to sample * `draw_samples`: number of samples to be drawn in the MCMC algorithms (warmup samples are ~2/3 of draw samples) -* `l2std`: standard deviation of BPINN predicition against L2 losses/Dataset -* `phystd`: standard deviation of BPINN predicition against Chosen Underlying ODE System -* `priorsNNw`: Vector of [mean, std] for BPINN parameter. Weights and Biases of BPINN are Normal Distributions by default +* `l2std`: standard deviation of BPINN prediction against L2 losses/Dataset +* `phystd`: standard deviation of BPINN prediction against Chosen Underlying ODE System +* `priorsNNw`: Tuple of (mean, std) for BPINN Network parameters. Weights and Biases of BPINN are Normal Distributions by default. * `param`: Vector of chosen ODE parameters Distributions in case of Inverse problems. * `autodiff`: Boolean Value for choice of Derivative Backend(default is numerical) * `physdt`: Timestep for approximating ODE in it's Time domain. (1/20.0 by default) - -# AdvancedHMC.jl is still developing convenience structs so might need changes on new releases. * `Kernel`: Choice of MCMC Sampling Algorithm (AdvancedHMC.jl implemenations HMC/NUTS/HMCDA) -* `Integratorkwargs`: A NamedTuple containing the chosen integrator and its keyword Arguments, as follows : - * `Integrator`: https://turinglang.org/AdvancedHMC.jl/stable/ - * `jitter_rate`: https://turinglang.org/AdvancedHMC.jl/stable/ - * `tempering_rate`: https://turinglang.org/AdvancedHMC.jl/stable/ -* `Adaptorkwargs`: A NamedTuple containing the chosen Adaptor, it's Metric and targetacceptancerate, as follows : - * `Adaptor`: https://turinglang.org/AdvancedHMC.jl/stable/ - * `Metric`: https://turinglang.org/AdvancedHMC.jl/stable/ - * `targetacceptancerate`: Target percentage(in decimal) of iterations in which the proposals were accepted(0.8 by default) +* `Integratorkwargs`: `Integrator`, `jitter_rate`, `tempering_rate`. Refer: https://turinglang.org/AdvancedHMC.jl/stable/ +* `Adaptorkwargs`: `Adaptor`, `Metric`, `targetacceptancerate`. Refer: https://turinglang.org/AdvancedHMC.jl/stable/ + Note: Target percentage(in decimal) of iterations in which the proposals are accepted (0.8 by default) * `MCMCargs`: A NamedTuple containing all the chosen MCMC kernel's(HMC/NUTS/HMCDA) Arguments, as follows : * `n_leapfrog`: number of leapfrog steps for HMC * `δ`: target acceptance probability for NUTS and HMCDA * `λ`: target trajectory length for HMCDA * `max_depth`: Maximum doubling tree depth (NUTS) * `Δ_max`: Maximum divergence during doubling tree (NUTS) + Refer: https://turinglang.org/AdvancedHMC.jl/stable/ * `progress`: controls whether to show the progress meter or not. * `verbose`: controls the verbosity. (Sample call args in AHMC) -""" +## Warnings -""" -dataset would be (x̂,t) -priors: pdf for W,b + pdf for ODE params +* AdvancedHMC.jl is still developing convenience structs so might need changes on new releases. """ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; strategy = GridTraining, dataset = [nothing], @@ -469,6 +447,7 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; progress = false, verbose = false, estim_collocate = false) + !(chain isa Lux.AbstractExplicitLayer) && (chain = Lux.transform(chain)) # NN parameter prior mean and variance(PriorsNN must be a tuple) if isinplace(prob) throw(error("The BPINN ODE solver only supports out-of-place ODE definitions, i.e. du=f(u,p,t).")) @@ -487,11 +466,11 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; throw(error("Dataset Required for Parameter Estimation.")) end - if chain isa Lux.AbstractExplicitLayer || chain isa Flux.Chain + if chain isa Lux.AbstractExplicitLayer # Flux-vector, Lux-Named Tuple initial_nnθ, recon, st = generate_Tar(chain, init_params) else - error("Only Lux.AbstractExplicitLayer and Flux.Chain neural networks are supported") + error("Only Lux.AbstractExplicitLayer Neural networks are supported") end if nchains > Threads.nthreads() @@ -501,13 +480,9 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; end # eltype(physdt) cause needs Float64 for find_good_stepsize - if chain isa Lux.AbstractExplicitLayer - # Lux chain(using component array later as vector_to_parameter need namedtuple) - initial_θ = collect(eltype(physdt), - vcat(ComponentArrays.ComponentArray(initial_nnθ))) - else - initial_θ = collect(eltype(physdt), initial_nnθ) - end + # Lux chain(using component array later as vector_to_parameter need namedtuple) + initial_θ = collect(eltype(physdt), + vcat(ComponentArrays.ComponentArray(initial_nnθ))) # adding ode parameter estimation nparameters = length(initial_θ) @@ -540,6 +515,10 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; end end + @info("Current Physics Log-likelihood : ", physloglikelihood(ℓπ, initial_θ)) + @info("Current Prior Log-likelihood : ", priorweights(ℓπ, initial_θ)) + @info("Current MSE against dataset Log-likelihood : ", L2LossData(ℓπ, initial_θ)) + Adaptor, Metric, targetacceptancerate = Adaptorkwargs[:Adaptor], Adaptorkwargs[:Metric], Adaptorkwargs[:targetacceptancerate] @@ -585,6 +564,13 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; Kernel = AdvancedHMC.make_kernel(MCMC_alg, integrator) samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, adaptor; progress = progress, verbose = verbose, drop_warmup = true) + + @info("Sampling Complete.") + @info("Current Physics Log-likelihood : ", physloglikelihood(ℓπ, samples[end])) + @info("Current Prior Log-likelihood : ", priorweights(ℓπ, samples[end])) + @info("Current MSE against dataset Log-likelihood : ", + L2LossData(ℓπ, samples[end])) + # return a chain(basic chain),samples and stats matrix_samples = reshape(hcat(samples...), (length(samples[1]), length(samples), 1)) mcmc_chain = MCMCChains.Chains(matrix_samples) From 3498ddc8264f5f71841ff86f118f98777b5cb284 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sun, 4 Feb 2024 02:47:52 +0530 Subject: [PATCH 15/53] keep new dir --- src/NeuralPDE.jl | 6 +++--- src/{ => bayesian}/BPINN_ode.jl | 0 src/{ => bayesian}/advancedHMC_MCMC.jl | 0 src/{ => bayesian}/collocated_estim.jl | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename src/{ => bayesian}/BPINN_ode.jl (100%) rename src/{ => bayesian}/advancedHMC_MCMC.jl (100%) rename src/{ => bayesian}/collocated_estim.jl (100%) diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 57d13d255e..66d6e9c58f 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -50,10 +50,10 @@ include("dae_solve.jl") include("transform_inf_integral.jl") include("discretize.jl") include("neural_adapter.jl") -include("advancedHMC_MCMC.jl") -include("BPINN_ode.jl") +include("bayesian/advancedHMC_MCMC.jl") +include("bayesian/BPINN_ode.jl") include("PDE_BPINN.jl") -include("collocated_estim.jl") +include("bayesian/collocated_estim.jl") export NNODE, TerminalPDEProblem, NNPDEHan, NNPDENS, NNRODE, NNDAE, KolmogorovPDEProblem, NNKolmogorov, NNStopping, ParamKolmogorovPDEProblem, diff --git a/src/BPINN_ode.jl b/src/bayesian/BPINN_ode.jl similarity index 100% rename from src/BPINN_ode.jl rename to src/bayesian/BPINN_ode.jl diff --git a/src/advancedHMC_MCMC.jl b/src/bayesian/advancedHMC_MCMC.jl similarity index 100% rename from src/advancedHMC_MCMC.jl rename to src/bayesian/advancedHMC_MCMC.jl diff --git a/src/collocated_estim.jl b/src/bayesian/collocated_estim.jl similarity index 100% rename from src/collocated_estim.jl rename to src/bayesian/collocated_estim.jl From 8ba64b2ea633b9fb694e68cc08cb7d056417df62 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Mon, 12 Feb 2024 19:40:25 +0530 Subject: [PATCH 16/53] having problems with eval() call in recursive Dict creation --- src/PDE_BPINN.jl | 203 +++++++------- src/training_strategies.jl | 36 ++- test/BPINN_PDEinvsol_tests.jl | 502 +++++++++++++++++++++++++++++++++- 3 files changed, 631 insertions(+), 110 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index b01df40a9f..96950961ff 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -63,28 +63,86 @@ mutable struct PDELogTargetDensity{ end end +# for bc case, [bc]/bc eqs must be passed along with dataset_bc[i] +# and final loss for bc must be together in a vector(bcs has seperate type of dataset_bc) +# eqs is vector of pde eqs and dataset here is dataset_pde +# normally you get vector of losses +function get_lossy(pinnrep, dataset, eqs) + depvars = pinnrep.depvars # order is same as dataset and interps + dict_depvar_input = pinnrep.dict_depvar_input + + Dict_differentials0 = Dict() + exp = toexpr(eqs) + Symbolics.variable.(hcat(pinnrep.indvars, pinnrep.depvars)) + for exp_i in exp + recur_expression(exp_i, Dict_differentials0) + end + # Dict_differentials is now filled with Differential operator => diff_i key-value pairs + + Dict_differentials = Dict() + for (a, b) in Dict_differentials0 + # println(eval(a.args[1])) + # Symbolics.operation(Symbolics.value(z)) + println(a) + a = Symbolics.parse_expr_to_symbolic(a, NeuralPDE) + Dict_differentials[a] = b + end + + # masking operation + println("Dict_differentials : ", Dict_differentials) + a = substitute.(eqs, Ref(Dict_differentials)) + println("Masked Differential term : ", a) + + to_subs, tobe_subs = get_symbols(dataset, depvars, eqs) + # for each row in dataset create u values for substituing in equation, n_equations=n_rows + eq_subs = [Dict(tobe_subs[depvar] => to_subs[depvar][i] for depvar in depvars) + for i in 1:size(dataset[1][:, 1])[1]] + + b = [] + for eq_sub in eq_subs + push!(b, [substitute(a_i, eq_sub) for a_i in a]) + end + + # reverse dict for re-substituing values of Differential(t)(u(t)) etc + rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) + + c = [] + for b_i in b + push!(c, substitute.(b_i, Ref(rev_Dict_differentials))) + end + println("After re Substituting depvars : ", c[1]) + # c = hcat(c...) + + # get losses + loss_functions = [[build_loss_function(pinnrep, eq, pde_indvar) + for (eq, pde_indvar, integration_indvar) in zip(c[i], + pinnrep.pde_indvars, + pinnrep.pde_integration_vars)] for i in eachindex(c)] + + return loss_functions +end + # dataset_pde has normal matrix format # dataset_bc has format of Vector{typeof(dataset_pde )} as each bc has different domain requirements -function get_symbols(dict_depvar_input, dataset, depvars) - # get datasets into splattable form - splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] - # splat datasets onto Linear interpolations tables - interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] - interps = Dict(depvars .=> interps) +function get_symbols(dataset, depvars, eqs) + depvar_vals = [dataset_i[:, 1] for dataset_i in dataset] + # order of depvars + to_subs = Dict(depvars .=> depvar_vals) - Dict_symbol_interps = Dict(depvar => (interps[depvar], dict_depvar_input[depvar]) - for depvar in depvars) + asrt = Symbolics.get_variables.(eqs) + # want only symbols of depvars + temp = unique(reduce(vcat, asrt)) tobe_subs = Dict() - for (a, b) in dict_depvar_input - tobe_subs[a] = eval(:($a($(b...)))) + for a in depvars + for i in temp + expr = toexpr(i) + if (expr isa Expr) && (expr.args[1] == a) + tobe_subs[a] = i + end + end end - to_subs = Dict() - for (a, b) in Dict_symbol_interps - b1, b2 = b - to_subs[a] = eval(:($b1($(b2...)))) - end return to_subs, tobe_subs end @@ -98,7 +156,9 @@ function recur_expression(exp, Dict_differentials) # first symbol of differential term # Dict_differentials for masking differential terms # and resubstituting differentials in equations after putting in interpolations - Dict_differentials[eval(in_exp)] = Symbol("diff_$(length(Dict_differentials)+1)") + println("starting") + Dict_differentials[in_exp] = Symbolics.variable("diff_$(length(Dict_differentials)+1)") + println("ending") return else @@ -107,71 +167,11 @@ function recur_expression(exp, Dict_differentials) end end -# get datafree loss functions for new loss type -# need to call merge_strategy_with_loss_function() variant after this -function merge_dataset_with_loss_function(pinnrep::NeuralPDE.PINNRepresentation, - dataset, - datafree_pde_loss_function, - datafree_bc_loss_function) - @unpack domains, eqs, bcs, dict_indvars, dict_depvars, flat_init_params = pinnrep - - eltypeθ = eltype(pinnrep.flat_init_params) - - train_sets = [[dataset[i][:, 2] for i in eachindex(dataset)], [[0;;], [0;;], [0;;]]] - - # the points in the domain and on the boundary - pde_train_sets, bcs_train_sets = train_sets - pde_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), - pde_train_sets) - bcs_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), - bcs_train_sets) - pde_loss_functions = [get_loss_function(_loss, _set, eltypeθ) - for (_loss, _set) in zip(datafree_pde_loss_function, - pde_train_sets)] - - bc_loss_functions = [get_loss_function(_loss, _set, eltypeθ) - for (_loss, _set) in zip(datafree_bc_loss_function, bcs_train_sets)] - - pde_loss_functions, bc_loss_functions -end - -function get_loss_function(loss_function, train_set, eltypeθ; τ = nothing) - loss = (θ) -> mean(abs2, loss_function(train_set, θ)) -end - -# for bc case, [bc]/bc eqs must be passed along with dataset_bc[i] -# and final loss for bc must be together in a vector(bcs has seperate type of dataset_bc) -# eqs is vector of pde eqs and dataset here is dataset_pde -# normally you get vector of losses -function get_loss_2(pinnrep, dataset, eqs) - depvars = pinnrep.depvars # order is same as dataset and interps - dict_depvar_input = pinnrep.dict_depvar_input - - to_subs, tobe_subs = get_symbols(dict_depvar_input, dataset, depvars) - interp_subs_dict = Dict(tobe_subs[depvar] => to_subs[depvar] for depvar in depvars) - - Dict_differentials = Dict() - exp = toexpr(eqs) - void_value = [recur_expression(exp_i, Dict_differentials) for exp_i in exp] - # Dict_differentials is now filled with Differential operator => diff_i key-value pairs - - # masking operation - a = substitute.(eqs, Ref(Dict_differentials)) - b = substitute.(a, Ref(interp_subs_dict)) - # reverse dict for re-substituing values of Differential(t)(u(t)) etc - rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) - eqs = substitute.(b, Ref(rev_Dict_differentials)) - # get losses - loss_functions = [NeuralPDE.build_loss_function(pinnrep, - eqs[i], - pinnrep.pde_indvars[i]) for i in eachindex(eqs)] -end - function LogDensityProblems.logdensity(Tar::PDELogTargetDensity, θ) # for parameter estimation neccesarry to use multioutput case return Tar.full_loglikelihood(setparameters(Tar, θ), - Tar.allstd) + priorlogpdf(Tar, θ) + Tar.L2_loss2(setparameters(Tar, θ), - Tar.allstd) + Tar.allstd) + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) + + Tar.L2_loss2(setparameters(Tar, θ), Tar.allstd) end function setparameters(Tar::PDELogTargetDensity, θ) @@ -218,7 +218,7 @@ function L2LossData(Tar::PDELogTargetDensity, θ) # dataset of form Vector[matrix_x, matrix_y, matrix_z] # matrix_i is of form [i,indvar1,indvar2,..] (needed in case if heterogenous domains) # note that indvar1,indvar2.. cols can be different values for different depvar matrices - # order follows pinnrep.depvars orders of variables (order of declaration in @variables macro) + # dataset,phi order follows pinnrep.depvars orders of variables (order of declaration in @variables macro) # Phi is the trial solution for each NN in chain array # Creating logpdf( MvNormal(Phi(t,θ),std), dataset[i] ) @@ -255,7 +255,7 @@ function priorlogpdf(Tar::PDELogTargetDensity, θ) return (invlogpdf + - logpdf(nnwparams, θ[1:(length(θ) - Tar.extraparams)])) + logpdf(nnwparams, θ[1:(length(θ) - Tar.extraparams)])) end return logpdf(nnwparams, θ) end @@ -403,28 +403,30 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; dataset_pde, dataset_bc = discretization.dataset eqs = pinnrep.eqs - yuh1 = get_loss_2(pinnrep, dataset_pde, eqs) - eqs = pinnrep.bcs - yuh2 = get_loss_2(pinnrep, dataset_bc, eqs) + yuh1 = get_lossy(pinnrep, dataset_pde, eqs) + # eqs = pinnrep.bcs + # yuh2 = get_lossy(pinnrep, dataset_pde, eqs) - pde_loss_functions, bc_loss_functions = merge_dataset_with_loss_function(pinnrep, - dataset, - yuh1, - yuh2) + pde_loss_functions = [merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, + GridTraining(0.1), + yuh1[i], + nothing; train_sets_pde = [data_pde[i, :] for data_pde in dataset_pde], + train_sets_bc = nothing)[1] + for i in eachindex(yuh1)] function L2_loss2(θ, allstd) stdpdes, stdbcs, stdextra = allstd - pde_loglikelihoods = [logpdf(Normal(0, stdpdes[i]), pde_loss_function(θ)) - for (i, pde_loss_function) in enumerate(pde_loss_functions)] - - bc_loglikelihoods = [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) - for (j, bc_loss_function) in enumerate(bc_loss_functions)] - println("pde_loglikelihoods : ", pde_loglikelihoods) - println("bc_loglikelihoods : ", bc_loglikelihoods) - return sum(sum(pde_loglikelihoods) + sum(bc_loglikelihoods)) + pde_loglikelihoods = [[logpdf(Normal(0, 0.8 * stdpdes[i]), pde_loss_function(θ)) + for (i, pde_loss_function) in enumerate(pde_loss_functions[i])] + for i in eachindex(pde_loss_functions)] + + # bc_loglikelihoods = [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) + # for (j, bc_loss_function) in enumerate(bc_loss_functions)] + # println("bc_loglikelihoods : ", bc_loglikelihoods) + return sum(sum(pde_loglikelihoods)) + # sum(sum(pde_loglikelihoods) + sum(bc_loglikelihoods)) end - println(L2_loss2) # WIP split dataset to respective equations if ((dataset_bc isa Nothing) && (dataset_pde isa Nothing)) dataset = nothing @@ -501,7 +503,8 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; names, ninv, initial_nnθ, - full_weighted_loglikelihood, L2_loss2, + full_weighted_loglikelihood, + L2_loss2, Φ) Adaptor, Metric, targetacceptancerate = Adaptorkwargs[:Adaptor], @@ -516,6 +519,9 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; ℓπ.allstd)) @info("Current Prior Log-likelihood : ", priorlogpdf(ℓπ, initial_θ)) @info("Current MSE against dataset Log-likelihood : ", L2LossData(ℓπ, initial_θ)) + @info("Current L2_LOSSY : ", + ℓπ.L2_loss2(setparameters(ℓπ, initial_θ), + ℓπ.allstd)) # parallel sampling option if nchains != 1 @@ -574,6 +580,9 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; @info("Current Prior Log-likelihood : ", priorlogpdf(ℓπ, samples[end])) @info("Current MSE against dataset Log-likelihood : ", L2LossData(ℓπ, samples[end])) + @info("Current L2_LOSSY : ", + ℓπ.L2_loss2(setparameters(ℓπ, samples[end]), + ℓπ.allstd)) fullsolution = BPINNstats(mcmc_chain, samples, stats) ensemblecurves, estimnnparams, estimated_params, timepoints = inference(samples, diff --git a/src/training_strategies.jl b/src/training_strategies.jl index a419afcdbf..d3e134e8a2 100644 --- a/src/training_strategies.jl +++ b/src/training_strategies.jl @@ -14,6 +14,27 @@ struct GridTraining{T} <: AbstractTrainingStrategy dx::T end +function get_dataset_train_points(eqs, train_sets, pinnrep) + dict_depvar_input = pinnrep.dict_depvar_input + depvars = pinnrep.depvars + dict_depvars = pinnrep.dict_depvars + dict_indvars = pinnrep.dict_indvars + + symbols_input = [(i, dict_depvar_input[i]) for i in depvars] + eq_args = NeuralPDE.get_argument(eqs, dict_indvars, dict_depvars) + points = [] + for eq_arg in eq_args + a = [] + for i in eachindex(symbols_input) + if symbols_input[i][2] == eq_arg + push!(a, train_sets[i][:, 2:end]') + end + end + push!(points, vcat(a...)) + end + return points +end + # include dataset points in pde_residual loglikelihood (BayesianPINN) function merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, strategy::GridTraining, @@ -25,7 +46,10 @@ function merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, # is vec as later each _set in pde_train_sets are coloumns as points transformed to vector of points (pde_train_sets must be rowwise) pde_loss_functions = if !(train_sets_pde isa Nothing) - pde_train_sets = [train_set[:, 2:end] for train_set in train_sets_pde] + + pde_train_sets = get_dataset_train_points(eqs, train_sets_pde, pinnrep) + println(" pde train set : ", pde_train_sets) + println("type pde train set : ", size(pde_train_sets)) pde_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), pde_train_sets) [get_loss_function(_loss, _set, eltypeθ, strategy) @@ -36,7 +60,10 @@ function merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, end bc_loss_functions = if !(train_sets_bc isa Nothing) - bcs_train_sets = [train_set[:, 2:end] for train_set in train_sets_bc] + + bcs_train_sets = get_dataset_train_points(bcs, train_sets_bc, pinnrep) + println("bcs train set : ", bcs_train_sets) + println("type bcs train set : ", size(bcs_train_sets)) bcs_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), bcs_train_sets) [get_loss_function(_loss, _set, eltypeθ, strategy) @@ -71,7 +98,10 @@ function merge_strategy_with_loss_function(pinnrep::PINNRepresentation, bc_loss_functions = [get_loss_function(_loss, _set, eltypeθ, strategy) for (_loss, _set) in zip(datafree_bc_loss_function, bcs_train_sets)] - + println("pde_train_sets : ",pde_train_sets) + println("pde_train_sets : ",size(pde_train_sets)) + println("bc_train_sets : ",bcs_train_sets) + println("bc_train_sets : ",size(bcs_train_sets)) pde_loss_functions, bc_loss_functions end diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index b340865bed..870a6aac53 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -35,8 +35,8 @@ Random.seed!(100) dataset = [hcat(u1, timepoints)] # checking all training strategies - discretization = BayesianPINN([chainl], StochasticTraining(200), param_estim = true, - dataset = [dataset, nothing]) + discretization = BayesianPINN([chainl], StochasticTraining(200), param_estim = true, + dataset = [dataset, nothing]) ahmc_bayesian_pinn_pde(pde_system, discretization; @@ -47,8 +47,8 @@ Random.seed!(100) saveats = [1 / 50.0], param = [LogNormal(6.0, 0.5)]) - discretization = BayesianPINN([chainl], QuasiRandomTraining(200), param_estim = true, - dataset = [dataset, nothing]) + discretization = BayesianPINN([chainl], QuasiRandomTraining(200), param_estim = true, + dataset = [dataset, nothing]) ahmc_bayesian_pinn_pde(pde_system, discretization; @@ -59,8 +59,8 @@ Random.seed!(100) saveats = [1 / 50.0], param = [LogNormal(6.0, 0.5)]) - discretization = BayesianPINN([chainl], QuadratureTraining(), param_estim = true, - dataset = [dataset, nothing]) + discretization = BayesianPINN([chainl], QuadratureTraining(), param_estim = true, + dataset = [dataset, nothing]) ahmc_bayesian_pinn_pde(pde_system, discretization; @@ -71,8 +71,8 @@ Random.seed!(100) saveats = [1 / 50.0], param = [LogNormal(6.0, 0.5)]) - discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, - dataset = [dataset, nothing]) + discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, + dataset = [dataset, nothing]) sol1 = ahmc_bayesian_pinn_pde(pde_system, discretization; @@ -132,8 +132,8 @@ end ts_ = hcat(sol(ts).t...)[1, :] dataset = [hcat(us[i, :], ts_) for i in 1:3] - discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, - dataset = [dataset, nothing]) + discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, + dataset = [dataset, nothing]) @named pde_system = PDESystem(eqs, bcs, domains, [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) @@ -153,3 +153,485 @@ end @test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] # @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] end + +@parameters t, x, p +@variables u(..) +Dt = Differential(t) +Dx = Differential(x) +eqs = [u(t, x) * Dt(u(t, x)) - cos(p * t) ~ 0, u(t, x) + Dx(u(t, x)) ~ 0.0] +bcs = [u(0, x) ~ 0.0, u(t, 10) ~ 1.0] +domains = [t ∈ Interval(0.0, 2.0), x ∈ Interval(0.0, 2.0)] + +chainl = Lux.Chain(Lux.Dense(2, 6, tanh), Lux.Dense(6, 1)) +initl, st = Lux.setup(Random.default_rng(), chainl) + +@named pde_system = PDESystem(eqs, + bcs, + domains, + [t, x], + [u(t, x)], + [p], + defaults = Dict([p => 4.0])) + +analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) +timepoints = collect(0.0:(1 / 100.0):2.0) +u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] +u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) +dataset = [hcat(u1, u1, timepoints)] + +# checking all training strategies +# discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, +# dataset = [dataset, nothing]) + +discretization = BayesianPINN([chainl], + GridTraining([0.2, 0.2]), + param_estim = true, dataset = [dataset, nothing]) + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05, 0.05], + phystd = [0.01, 0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0, 1 / 20.0], + param = [Normal(3.0, 0.5)], progress = true) + +param = 2 * π +ts = vec(sol1.timepoints[1]) +u_real = [analytic_sol_func1(0.0, t) for t in ts] +u_predict = pmean(sol1.ensemblesol[1]) + +@test u_predict≈u_real atol=1.5 +@test mean(u_predict .- u_real) < 0.1 +@test sol1.estimated_de_params[1]≈param atol=param * 0.3 + +# function get_symbols(dict_depvar_input, dataset, depvars, eqs) + +# # get datasets into splattable form +# splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] +# # splat datasets onto Linear interpolations tables +# interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] +# # this works as order of dataset matches order of depvars +# interps = Dict(depvars .=> interps) + +# Dict_symbol_interps = Dict(depvar => (interps[depvar], dict_depvar_input[depvar]) +# for depvar in depvars) + +# tobe_subs = Dict() + +# asrt = Symbolics.get_variables.(eqs) +# # want only symbols of depvars +# tempo = unique(reduce(vcat, asrt))[(end - length(depvars) + 1):end] +# # now we have all the depvars, we now need all depvars whcih can be substituted with data interps + +# tobe_subs = Dict() +# for a in depvars +# for i in tempo +# if toexpr(i).args[1] == a +# tobe_subs[a] = i +# end +# end +# end + +# # do the same thing as above here using pinnrep.indvars +# to_subs = Dict() +# for (a, b) in Dict_symbol_interps +# b1, b2 = b +# for i in tempo +# if toexpr(i).args[1] == a +# tobe_subs[a] = i +# end +# end +# end +# for (a, b) in Dict_symbol_interps +# b1, b2 = b +# to_subs[a] = eval(:($b1($(b2...)))) +# # Symbol("$b1($(b2...))") +# # eval(:($b1($(b2...)))) +# end + +# println("to_subs : ", to_subs) +# println("tobe_subs : ", tobe_subs) +# return to_subs, tobe_subs +# end + +# function recur_expression(exp, Dict_differentials) +# for in_exp in exp.args +# if !(in_exp isa Expr) +# # skip +,== symbols, characters etc +# continue + +# elseif in_exp.args[1] isa ModelingToolkit.Differential +# # first symbol of differential term +# # Dict_differentials for masking differential terms +# # and resubstituting differentials in equations after putting in interpolations +# temp = eval(in_exp) +# # println(" inside recursion : ") +# # println("in_exp went from ", in_exp, " to ", temp) +# # println("typeof in_exp went from ", typeof(in_exp), " to ", typeof(temp)) +# Dict_differentials[temp] = Symbol("diff_$(length(Dict_differentials)+1)") +# return + +# else +# recur_expression(in_exp, Dict_differentials) +# end +# end +# end + +# get datafree loss functions for new loss type +# need to call merge_strategy_with_loss_function() variant after this +function merge_dataset_with_loss_function(pinnrep::NeuralPDE.PINNRepresentation, + dataset, + datafree_pde_loss_function, + datafree_bc_loss_function) + @unpack domains, eqs, bcs, dict_indvars, dict_depvars, flat_init_params = pinnrep + + eltypeθ = eltype(pinnrep.flat_init_params) + + train_sets = [[dataset[i][:, 2] for i in eachindex(dataset)], [[0;;], [0;;], [0;;]]] + + # the points in the domain and on the boundary + pde_train_sets, bcs_train_sets = train_sets + # pde_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), + # pde_train_sets) + # bcs_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), + # bcs_train_sets) + pde_loss_functions = [get_loss_function(_loss, _set, eltypeθ) + for (_loss, _set) in zip(datafree_pde_loss_function, + pde_train_sets)] + + bc_loss_functions = [get_loss_function(_loss, _set, eltypeθ) + for (_loss, _set) in zip(datafree_bc_loss_function, bcs_train_sets)] + + pde_loss_functions, bc_loss_functions +end + +function get_loss_function(loss_function, train_set, eltypeθ; τ = nothing) + loss = (θ) -> mean(abs2, loss_function(train_set, θ)) +end + +# for bc case, [bc]/bc eqs must be passed along with dataset_bc[i] +# and final loss for bc must be together in a vector(bcs has seperate type of dataset_bc) +# eqs is vector of pde eqs and dataset here is dataset_pde +# normally you get vector of losses +function get_loss_2(pinnrep, dataset, eqs) + depvars = pinnrep.depvars # order is same as dataset and interps + dict_depvar_input = pinnrep.dict_depvar_input + + to_subs, tobe_subs = get_symbols(dict_depvar_input, dataset, depvars) + interp_subs_dict = Dict(tobe_subs[depvar] => to_subs[depvar] for depvar in depvars) + + Dict_differentials = Dict() + exp = toexpr(eqs) + void_value = [recur_expression(exp_i, Dict_differentials) for exp_i in exp] + # Dict_differentials is now filled with Differential operator => diff_i key-value pairs + + # masking operation + a = substitute.(eqs, Ref(Dict_differentials)) + println(a) + b = substitute.(a, Ref(interp_subs_dict)) + println(b) + # reverse dict for re-substituing values of Differential(t)(u(t)) etc + rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) + eqs = substitute.(b, Ref(rev_Dict_differentials)) + # get losses + loss_functions = [NeuralPDE.build_loss_function(pinnrep, + eqs[i], + pinnrep.pde_indvars[i]) for i in eachindex(eqs)] +end + +# -----------------=============== +eqs +a = substitute.(eqs, Ref(Dict(t => 1))) + +# after masking +# this can remove interpolations need +b = substitute.(eqs, Ref(Dict(u(t) => interp([1]...)))) + +toexpr(a[1]).args[2].args[2].args[2](3) +Symbol("$(u)") + +interp = LinearInterpolation([1, 2], [1, 23]) +typeof(interp) +LinearInterpolation{Vector{Int64}, Vector{Int64}, true, Int64} + +typeof(interp(t)) +SymbolicUtils.BasicSymbolic{Real} +interp_vars = [t] +interp(interp_vars...) +arg = pinnrep.dict_depvar_input[:u] +arg = [g, l] +pinnrep.indvars +@parameters (arg...) +eval(:($interp($(arg...)))) +b = substitute(a, Dict(t => 1)) +@parameters aa[1:2] +aa = [m, l] +l +m + +# >why not mask differential +function get_lossy(pinnrep, dataset, eqs) + depvars = pinnrep.depvars # order is same as dataset and interps + dict_depvar_input = pinnrep.dict_depvar_input + + Dict_differentials = Dict() + exp = toexpr(eqs) + for exp_i in exp + recur_expression(exp_i, Dict_differentials) + end + # Dict_differentials is now filled with Differential operator => diff_i key-value pairs + + # masking operation + println("Dict_differentials : ", Dict_differentials) + a = substitute.(eqs, Ref(Dict_differentials)) + println("Masked Differential term : ", a) + + to_subs, tobe_subs = get_symbols(dict_depvar_input, dataset, depvars, eqs) + # for each row in dataset create u values for substituing in equation, n_equations=n_rows + eq_subs = [Dict(tobe_subs[depvar] => to_subs[depvar][i] for depvar in depvars) + for i in 1:size(dataset[1][:, 1])[1]] + + b = [] + for eq_sub in eq_subs + push!(b, [substitute(a_i, eq_sub) for a_i in a]) + end + + # reverse dict for re-substituing values of Differential(t)(u(t)) etc + rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) + + c = [] + for b_i in b + push!(c, substitute.(b_i, Ref(rev_Dict_differentials))) + end + println("After re Substituting depvars : ", c[1]) + # c = vcat(c...) + println(c) + c + # get losses + # loss_functions = [NeuralPDE.build_loss_function(pinnrep, + # c[i, :][j], + # pinnrep.pde_indvars[j]) for j in eachindex(pinnrep.pde_indvars)] + # return loss_functions +end + +# finally dataset to be fed +# train sets format [[],[]] +pinnrep.pde_indvars +pinnrep = NeuralPDE.symbolic_discretize(pde_system, discretization) +eqs = pinnrep.eqs +yuh1 = get_lossy(pinnrep, dataset, eqs) +pde_loss_functions = [NeuralPDE.merge_strategy_with_loglikelihood_function(pinnrep, + GridTraining(0.1), + yuh1[i], + nothing; train_sets_pde = [data_pde[i, :] for data_pde in dataset], + train_sets_bc = nothing)[1] + for i in eachindex(yuh1)] +function L2_loss2(θ, allstd) + stdpdes, stdbcs, stdextra = allstd + pde_loglikelihoods = [[logpdf(Normal(0, 0.8 * stdpdes[i]), pde_loss_function(θ)) + for (i, pde_loss_function) in enumerate(pde_loss_functions[i])] + for i in eachindex(pde_loss_functions)] + + # bc_loglikelihoods = [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) + # for (j, bc_loss_function) in enumerate(bc_loss_functions)] + # println("bc_loglikelihoods : ", bc_loglikelihoods) + return sum(sum(pde_loglikelihoods)) + # sum(sum(pde_loglikelihoods) + sum(bc_loglikelihoods)) +end + +L2_loss2([1, 2, 3, 4], [1, 1, 1]) + +[NeuralPDE.parse_equation(pinnrep, exa) for exa in exam] +a = "diff_1" +substitute(a * u(t, x) - cos(p * t) ~ 0, Dict(u(t, x) => 1.0)) +substitute(eqs[1], Dict(u(t, x) => 1.0)) +# dataset_pde has normal matrix format +# dataset_bc has format of Vector{typeof(dataset_pde )} as each bc has different domain requirements +function get_symbols(dict_depvar_input, dataset, depvars, eqs) + depvar_vals = [dataset_i[:, 1] for dataset_i in dataset] + # order of depvars + to_subs = Dict(pinnrep.depvars .=> depvar_vals) + + asrt = Symbolics.get_variables.(eqs) + # want only symbols of depvars + temp = unique(reduce(vcat, asrt)) + # now we have all the depvars, we now need all depvars whcih can be substituted with data interps + + tobe_subs = Dict() + for a in depvars + for i in temp + expr = toexpr(i) + if (expr isa Expr) && (expr.args[1] == a) + tobe_subs[a] = i + end + end + end + + return to_subs, tobe_subs +end + +yuh = get_symbols(pinnrep.dict_depvar_input, dataset, pinnrep.depvars, pinnrep.eqs) + +function recur_expression(exp, Dict_differentials) + for in_exp in exp.args + if !(in_exp isa Expr) + # skip +,== symbols, characters etc + continue + + elseif in_exp.args[1] isa ModelingToolkit.Differential + # first symbol of differential term + # Dict_differentials for masking differential terms + # and resubstituting differentials in equations after putting in interpolations + # temp = in_exp.args[end] + # in_exp.args[end] = Symbolics.variable(in_exp.args[end]) + + Dict_differentials[in_exp] = Symbolics.variable("diff_$(length(Dict_differentials)+1)") + return + else + recur_expression(in_exp, Dict_differentials) + end + end +end +vars = Symbolics.variable.(hcat(pinnrep.indvars, pinnrep.depvars)) +toexpr(Differential(t)(Differential(u)(u(t))) + u(t) ~ 0).args[2] +eqs +# Differential(t)(u(t)) - cos(p * t) ~ 0 +exprs = toexpr(eqs) +pop = Dict() +recur_expression(exprs, pop) +pop1 = Dict() +for (a, b) in pop + pop1[eval(a)] = b +end +pop1 +a = substitute(eqs, pop1) + +transpose(dataset[1]) +pde_system.eqs +pde_system.bcs +eqs = pde_system.eqs +Symbolics.get_variables(eqs[1]) +# eqs=a + +NeuralPDE.get_variables(pinnrep.eqs, pinnrep.dict_indvars, pinnrep.dict_depvars) +NeuralPDE.get_argument(pinnrep.bcs, pinnrep.dict_indvars, pinnrep.dict_depvars) +dx = pinnrep.strategy.dx +eltypeθ = eltype(pinnrep.flat_init_params) + +# solve dataset physics loss for heterogenous case +# create number of equations as number of interpolation and points(n rows) +# follow masking and finally feed training sets as set in interpolations input of u(t,x,..) + +# logic for recursion formula to parse differentials +# # this below has the whole differential term +toexpr(pde_system.eqs[1]).args[2].args[3].args[3] +# toexpr(pde_system.eqs[1]).args[2].args[3].args[3] +# # .args[1] isa ModelingToolkit.Differential + +# logic for interpolation and indvars splatting to get Equation parsing terms +# splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] +# # splat datasets onto Linear interpolations tables +# interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] +# interps = Dict(depvars .=> interps) +# get datasets into splattable form +# splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] +# # splat datasets onto Linear interpolations tables +# yu = [LinearInterpolation(splat_i...) for splat_i in splat_form] +# Symbol(:($(yu[1]))) + +# logic to contrauct dict to feed for masking +# Dict(interps[depvar] => dict_depvar_input[depvar] for depvar in depvars) + +# what do i want? +# > what do i have? +# i have a dataset of depvars and corresponding indvars values +# i want for each equation indvars - get_variables() +# construct physics losses based on above list and dataset values +# dataset - dict_depvars_input construct +# use this on dataset + +# from pinnrep and dataset gives eqaution wise datasets +symbols_input = [(i, pinnrep.dict_depvar_input[i]) for i in pinnrep.depvars] +eq_args = NeuralPDE.get_argument(eqs, pinnrep.dict_indvars, pinnrep.dict_depvars) +points = [] +for eq_arg in eq_args + a = [] + for i in eachindex(symbols_input) + if symbols_input[i][2] == eq_arg + push!(a, dataset[i][2:end]) + end + end + push!(points, a) +end +typeof(points[1]) + +d = Dict() +dataset[1][:, 2:end]' +Dict(symbols_input[1][2] .=> dataset[1][:, 2:end]') +symbols_input[1][2] .= dataset[1][:, 2:end] +for m in symbols_input + d[m[2]] .= dataset[i][:, 2] +end +d +for i in eachindex(dataset) + dataset[i] + # depvars[i] +end + +toexpr(pde_system.eqs) +pinnrep. + +@parameterst, p +@variables u(..) + +Dt = Differential(t) +eqs = Dt(u(t)) - cos(p * t) ~ 0 +bcs = [u(0) ~ 0.0] +domains = [t ∈ Interval(0.0, 2.0)] + +chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) +initl, st = Lux.setup(Random.default_rng(), chainl) + +@named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [u(t)], + [p], + defaults = Dict([p => 4.0])) + +analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) +timepoints = collect(0.0:(1 / 100.0):2.0) +u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] +u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) +dataset = [hcat(u1, timepoints)] + +# checking all training strategies +discretization = BayesianPINN([chainl], GridTraining(0.01), param_estim = true, + dataset = [dataset, nothing]) + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(6.0, 0.5)], progress = true) +Symoblics.value(a) +ex = :(y(t) ~ x(t)) +parse_expr_to_symbolic(ex[1], Main) # gives the symbolic expression `y(t) ~ x(t)` in empty Main + +# Now do a whole system + +ex = [:(y ~ x) + :(y ~ -2x + 3 / z) + :(z ~ 2)] +eqs = parse_expr_to_symbolic.(ex, (Main,)) + +@variables x y z +ex = [y ~ x + y ~ -2x + 3 / z + z ~ 2] +all(isequal.(eqs, ex)) # true \ No newline at end of file From aa15410ef0533fa25da815774150a8ad47628b90 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Mon, 12 Feb 2024 19:44:39 +0530 Subject: [PATCH 17/53] removed bayesian folder --- src/{bayesian => }/BPINN_ode.jl | 0 src/NeuralPDE.jl | 4 ++-- src/{bayesian => }/advancedHMC_MCMC.jl | 0 src/{bayesian => }/collocated_estim.jl | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename src/{bayesian => }/BPINN_ode.jl (100%) rename src/{bayesian => }/advancedHMC_MCMC.jl (100%) rename src/{bayesian => }/collocated_estim.jl (100%) diff --git a/src/bayesian/BPINN_ode.jl b/src/BPINN_ode.jl similarity index 100% rename from src/bayesian/BPINN_ode.jl rename to src/BPINN_ode.jl diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 66d6e9c58f..de20f9a88f 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -50,8 +50,8 @@ include("dae_solve.jl") include("transform_inf_integral.jl") include("discretize.jl") include("neural_adapter.jl") -include("bayesian/advancedHMC_MCMC.jl") -include("bayesian/BPINN_ode.jl") +include("advancedHMC_MCMC.jl") +include("BPINN_ode.jl") include("PDE_BPINN.jl") include("bayesian/collocated_estim.jl") diff --git a/src/bayesian/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl similarity index 100% rename from src/bayesian/advancedHMC_MCMC.jl rename to src/advancedHMC_MCMC.jl diff --git a/src/bayesian/collocated_estim.jl b/src/collocated_estim.jl similarity index 100% rename from src/bayesian/collocated_estim.jl rename to src/collocated_estim.jl From 9475d27ccdc8e22f110e2298f5c49dde9c818920 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Mon, 12 Feb 2024 19:58:41 +0530 Subject: [PATCH 18/53] cleaned files, removed DataInterpolations --- Project.toml | 1 - src/NeuralPDE.jl | 3 +- test/BPINN_PDEinvsol_tests.jl | 229 +--------------------------------- 3 files changed, 3 insertions(+), 230 deletions(-) diff --git a/Project.toml b/Project.toml index 3f82c4de90..fd683ede3b 100644 --- a/Project.toml +++ b/Project.toml @@ -10,7 +10,6 @@ ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" Cubature = "667455a9-e2ce-5579-9412-b964f529a492" -DataInterpolations = "82cc6244-b520-54b8-b5a6-8a565e85f1d0" DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" DiffEqNoiseProcess = "77a26b50-5914-5dd7-bc55-306e6241c503" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index de20f9a88f..84fb852be9 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -25,7 +25,6 @@ using Symbolics: wrap, unwrap, arguments, operation using SymbolicUtils using AdvancedHMC, LogDensityProblems, LinearAlgebra, Functors, MCMCChains using MonteCarloMeasurements -using DataInterpolations: LinearInterpolation import ModelingToolkit: value, nameof, toexpr, build_expr, expand_derivatives import DomainSets: Domain, ClosedInterval import ModelingToolkit: Interval, infimum, supremum #,Ball @@ -53,7 +52,7 @@ include("neural_adapter.jl") include("advancedHMC_MCMC.jl") include("BPINN_ode.jl") include("PDE_BPINN.jl") -include("bayesian/collocated_estim.jl") +include("collocated_estim.jl") export NNODE, TerminalPDEProblem, NNPDEHan, NNPDENS, NNRODE, NNDAE, KolmogorovPDEProblem, NNKolmogorov, NNStopping, ParamKolmogorovPDEProblem, diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 870a6aac53..0a23cfa669 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -205,111 +205,6 @@ u_predict = pmean(sol1.ensemblesol[1]) @test mean(u_predict .- u_real) < 0.1 @test sol1.estimated_de_params[1]≈param atol=param * 0.3 -# function get_symbols(dict_depvar_input, dataset, depvars, eqs) - -# # get datasets into splattable form -# splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] -# # splat datasets onto Linear interpolations tables -# interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] -# # this works as order of dataset matches order of depvars -# interps = Dict(depvars .=> interps) - -# Dict_symbol_interps = Dict(depvar => (interps[depvar], dict_depvar_input[depvar]) -# for depvar in depvars) - -# tobe_subs = Dict() - -# asrt = Symbolics.get_variables.(eqs) -# # want only symbols of depvars -# tempo = unique(reduce(vcat, asrt))[(end - length(depvars) + 1):end] -# # now we have all the depvars, we now need all depvars whcih can be substituted with data interps - -# tobe_subs = Dict() -# for a in depvars -# for i in tempo -# if toexpr(i).args[1] == a -# tobe_subs[a] = i -# end -# end -# end - -# # do the same thing as above here using pinnrep.indvars -# to_subs = Dict() -# for (a, b) in Dict_symbol_interps -# b1, b2 = b -# for i in tempo -# if toexpr(i).args[1] == a -# tobe_subs[a] = i -# end -# end -# end -# for (a, b) in Dict_symbol_interps -# b1, b2 = b -# to_subs[a] = eval(:($b1($(b2...)))) -# # Symbol("$b1($(b2...))") -# # eval(:($b1($(b2...)))) -# end - -# println("to_subs : ", to_subs) -# println("tobe_subs : ", tobe_subs) -# return to_subs, tobe_subs -# end - -# function recur_expression(exp, Dict_differentials) -# for in_exp in exp.args -# if !(in_exp isa Expr) -# # skip +,== symbols, characters etc -# continue - -# elseif in_exp.args[1] isa ModelingToolkit.Differential -# # first symbol of differential term -# # Dict_differentials for masking differential terms -# # and resubstituting differentials in equations after putting in interpolations -# temp = eval(in_exp) -# # println(" inside recursion : ") -# # println("in_exp went from ", in_exp, " to ", temp) -# # println("typeof in_exp went from ", typeof(in_exp), " to ", typeof(temp)) -# Dict_differentials[temp] = Symbol("diff_$(length(Dict_differentials)+1)") -# return - -# else -# recur_expression(in_exp, Dict_differentials) -# end -# end -# end - -# get datafree loss functions for new loss type -# need to call merge_strategy_with_loss_function() variant after this -function merge_dataset_with_loss_function(pinnrep::NeuralPDE.PINNRepresentation, - dataset, - datafree_pde_loss_function, - datafree_bc_loss_function) - @unpack domains, eqs, bcs, dict_indvars, dict_depvars, flat_init_params = pinnrep - - eltypeθ = eltype(pinnrep.flat_init_params) - - train_sets = [[dataset[i][:, 2] for i in eachindex(dataset)], [[0;;], [0;;], [0;;]]] - - # the points in the domain and on the boundary - pde_train_sets, bcs_train_sets = train_sets - # pde_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), - # pde_train_sets) - # bcs_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), - # bcs_train_sets) - pde_loss_functions = [get_loss_function(_loss, _set, eltypeθ) - for (_loss, _set) in zip(datafree_pde_loss_function, - pde_train_sets)] - - bc_loss_functions = [get_loss_function(_loss, _set, eltypeθ) - for (_loss, _set) in zip(datafree_bc_loss_function, bcs_train_sets)] - - pde_loss_functions, bc_loss_functions -end - -function get_loss_function(loss_function, train_set, eltypeθ; τ = nothing) - loss = (θ) -> mean(abs2, loss_function(train_set, θ)) -end - # for bc case, [bc]/bc eqs must be passed along with dataset_bc[i] # and final loss for bc must be together in a vector(bcs has seperate type of dataset_bc) # eqs is vector of pde eqs and dataset here is dataset_pde @@ -340,36 +235,6 @@ function get_loss_2(pinnrep, dataset, eqs) pinnrep.pde_indvars[i]) for i in eachindex(eqs)] end -# -----------------=============== -eqs -a = substitute.(eqs, Ref(Dict(t => 1))) - -# after masking -# this can remove interpolations need -b = substitute.(eqs, Ref(Dict(u(t) => interp([1]...)))) - -toexpr(a[1]).args[2].args[2].args[2](3) -Symbol("$(u)") - -interp = LinearInterpolation([1, 2], [1, 23]) -typeof(interp) -LinearInterpolation{Vector{Int64}, Vector{Int64}, true, Int64} - -typeof(interp(t)) -SymbolicUtils.BasicSymbolic{Real} -interp_vars = [t] -interp(interp_vars...) -arg = pinnrep.dict_depvar_input[:u] -arg = [g, l] -pinnrep.indvars -@parameters (arg...) -eval(:($interp($(arg...)))) -b = substitute(a, Dict(t => 1)) -@parameters aa[1:2] -aa = [m, l] -l -m - # >why not mask differential function get_lossy(pinnrep, dataset, eqs) depvars = pinnrep.depvars # order is same as dataset and interps @@ -415,37 +280,6 @@ function get_lossy(pinnrep, dataset, eqs) # return loss_functions end -# finally dataset to be fed -# train sets format [[],[]] -pinnrep.pde_indvars -pinnrep = NeuralPDE.symbolic_discretize(pde_system, discretization) -eqs = pinnrep.eqs -yuh1 = get_lossy(pinnrep, dataset, eqs) -pde_loss_functions = [NeuralPDE.merge_strategy_with_loglikelihood_function(pinnrep, - GridTraining(0.1), - yuh1[i], - nothing; train_sets_pde = [data_pde[i, :] for data_pde in dataset], - train_sets_bc = nothing)[1] - for i in eachindex(yuh1)] -function L2_loss2(θ, allstd) - stdpdes, stdbcs, stdextra = allstd - pde_loglikelihoods = [[logpdf(Normal(0, 0.8 * stdpdes[i]), pde_loss_function(θ)) - for (i, pde_loss_function) in enumerate(pde_loss_functions[i])] - for i in eachindex(pde_loss_functions)] - - # bc_loglikelihoods = [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) - # for (j, bc_loss_function) in enumerate(bc_loss_functions)] - # println("bc_loglikelihoods : ", bc_loglikelihoods) - return sum(sum(pde_loglikelihoods)) - # sum(sum(pde_loglikelihoods) + sum(bc_loglikelihoods)) -end - -L2_loss2([1, 2, 3, 4], [1, 1, 1]) - -[NeuralPDE.parse_equation(pinnrep, exa) for exa in exam] -a = "diff_1" -substitute(a * u(t, x) - cos(p * t) ~ 0, Dict(u(t, x) => 1.0)) -substitute(eqs[1], Dict(u(t, x) => 1.0)) # dataset_pde has normal matrix format # dataset_bc has format of Vector{typeof(dataset_pde )} as each bc has different domain requirements function get_symbols(dict_depvar_input, dataset, depvars, eqs) @@ -507,10 +341,6 @@ end pop1 a = substitute(eqs, pop1) -transpose(dataset[1]) -pde_system.eqs -pde_system.bcs -eqs = pde_system.eqs Symbolics.get_variables(eqs[1]) # eqs=a @@ -543,46 +373,7 @@ toexpr(pde_system.eqs[1]).args[2].args[3].args[3] # logic to contrauct dict to feed for masking # Dict(interps[depvar] => dict_depvar_input[depvar] for depvar in depvars) -# what do i want? -# > what do i have? -# i have a dataset of depvars and corresponding indvars values -# i want for each equation indvars - get_variables() -# construct physics losses based on above list and dataset values -# dataset - dict_depvars_input construct -# use this on dataset - -# from pinnrep and dataset gives eqaution wise datasets -symbols_input = [(i, pinnrep.dict_depvar_input[i]) for i in pinnrep.depvars] -eq_args = NeuralPDE.get_argument(eqs, pinnrep.dict_indvars, pinnrep.dict_depvars) -points = [] -for eq_arg in eq_args - a = [] - for i in eachindex(symbols_input) - if symbols_input[i][2] == eq_arg - push!(a, dataset[i][2:end]) - end - end - push!(points, a) -end -typeof(points[1]) - -d = Dict() -dataset[1][:, 2:end]' -Dict(symbols_input[1][2] .=> dataset[1][:, 2:end]') -symbols_input[1][2] .= dataset[1][:, 2:end] -for m in symbols_input - d[m[2]] .= dataset[i][:, 2] -end -d -for i in eachindex(dataset) - dataset[i] - # depvars[i] -end - -toexpr(pde_system.eqs) -pinnrep. - -@parameterst, p +@parameters t, p @variables u(..) Dt = Differential(t) @@ -618,20 +409,4 @@ sol1 = ahmc_bayesian_pinn_pde(pde_system, phystd = [0.01], l2std = [0.01], priorsNNw = (0.0, 1.0), saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)], progress = true) -Symoblics.value(a) -ex = :(y(t) ~ x(t)) -parse_expr_to_symbolic(ex[1], Main) # gives the symbolic expression `y(t) ~ x(t)` in empty Main - -# Now do a whole system - -ex = [:(y ~ x) - :(y ~ -2x + 3 / z) - :(z ~ 2)] -eqs = parse_expr_to_symbolic.(ex, (Main,)) - -@variables x y z -ex = [y ~ x - y ~ -2x + 3 / z - z ~ 2] -all(isequal.(eqs, ex)) # true \ No newline at end of file + param = [LogNormal(6.0, 0.5)], progress = true) \ No newline at end of file From b50989a95f60c37c497fc4bb9e9cfe1eb30bfb3c Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 15 Feb 2024 02:52:44 +0530 Subject: [PATCH 19/53] done with implementation --- src/PDE_BPINN.jl | 72 ++----- src/discretize.jl | 8 +- src/training_strategies.jl | 32 ++- test/BPINN_PDEinvsol_tests.jl | 362 ++++++++++++---------------------- 4 files changed, 163 insertions(+), 311 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index d53110e20d..7cbe22a804 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -67,53 +67,36 @@ end # and final loss for bc must be together in a vector(bcs has seperate type of dataset_bc) # eqs is vector of pde eqs and dataset here is dataset_pde # normally you get vector of losses -function get_lossy(pinnrep, dataset, eqs) +function get_lossy(pinnrep, dataset, Dict_differentials) + eqs = pinnrep.eqs depvars = pinnrep.depvars # order is same as dataset and interps - dict_depvar_input = pinnrep.dict_depvar_input - - Dict_differentials0 = Dict() - exp = toexpr(eqs) - Symbolics.variable.(hcat(pinnrep.indvars, pinnrep.depvars)) - for exp_i in exp - recur_expression(exp_i, Dict_differentials0) - end - # Dict_differentials is now filled with Differential operator => diff_i key-value pairs - - Dict_differentials = Dict() - for (a, b) in Dict_differentials0 - # println(eval(a.args[1])) - # Symbolics.operation(Symbolics.value(z)) - println(a) - a = Symbolics.parse_expr_to_symbolic(a, NeuralPDE) - Dict_differentials[a] = b - end + # Dict_differentials is filled with Differential operator => diff_i key-value pairs # masking operation - println("Dict_differentials : ", Dict_differentials) - a = substitute.(eqs, Ref(Dict_differentials)) - println("Masked Differential term : ", a) + eqs_new = substitute.(eqs, Ref(Dict_differentials)) to_subs, tobe_subs = get_symbols(dataset, depvars, eqs) # for each row in dataset create u values for substituing in equation, n_equations=n_rows eq_subs = [Dict(tobe_subs[depvar] => to_subs[depvar][i] for depvar in depvars) for i in 1:size(dataset[1][:, 1])[1]] + # for each point(eq_sub dictionary) substiute in all equations(eqs_new - masked equations) b = [] for eq_sub in eq_subs - push!(b, [substitute(a_i, eq_sub) for a_i in a]) + push!(b, [substitute(eq, eq_sub) for eq in eqs_new]) end + # now we have vector of equation vectors # reverse dict for re-substituing values of Differential(t)(u(t)) etc rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) + # for each vector in vecvtor of equation vectorbroadcast resubstituing OG mask values c = [] for b_i in b push!(c, substitute.(b_i, Ref(rev_Dict_differentials))) end - println("After re Substituting depvars : ", c[1]) - # c = hcat(c...) - # get losses + # get losses, zip each equation with args for each build_loss call per equation vector loss_functions = [[build_loss_function(pinnrep, eq, pde_indvar) for (eq, pde_indvar, integration_indvar) in zip(c[i], pinnrep.pde_indvars, @@ -146,27 +129,6 @@ function get_symbols(dataset, depvars, eqs) return to_subs, tobe_subs end -function recur_expression(exp, Dict_differentials) - for in_exp in exp.args - if !(in_exp isa Expr) - # skip +,== symbols, characters etc - continue - - elseif in_exp.args[1] isa ModelingToolkit.Differential - # first symbol of differential term - # Dict_differentials for masking differential terms - # and resubstituting differentials in equations after putting in interpolations - println("starting") - Dict_differentials[in_exp] = Symbolics.variable("diff_$(length(Dict_differentials)+1)") - println("ending") - return - - else - recur_expression(in_exp, Dict_differentials) - end - end -end - function LogDensityProblems.logdensity(Tar::PDELogTargetDensity, θ) # for parameter estimation neccesarry to use multioutput case return Tar.full_loglikelihood(setparameters(Tar, θ), @@ -398,31 +360,33 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; Adaptorkwargs = (Adaptor = StanHMCAdaptor, Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), Integratorkwargs = (Integrator = Leapfrog,), saveats = [1 / 10.0], - numensemble = floor(Int, draw_samples / 3), progress = false, verbose = false) + numensemble = floor(Int, draw_samples / 3), Dict_differentials = Dict(), + progress = false, verbose = false) pinnrep = symbolic_discretize(pde_system, discretization) dataset_pde, dataset_bc = discretization.dataset - eqs = pinnrep.eqs - yuh1 = get_lossy(pinnrep, dataset_pde, eqs) + yuh1 = get_lossy(pinnrep, dataset_pde, Dict_differentials) # eqs = pinnrep.bcs # yuh2 = get_lossy(pinnrep, dataset_pde, eqs) + # this is a vector of tuple{vector,nothing} pde_loss_functions = [merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, GridTraining(0.1), yuh1[i], nothing; train_sets_pde = [data_pde[i, :] for data_pde in dataset_pde], - train_sets_bc = nothing)[1] + train_sets_bc = nothing) for i in eachindex(yuh1)] function L2_loss2(θ, allstd) stdpdes, stdbcs, stdextra = allstd - pde_loglikelihoods = [[logpdf(Normal(0, 0.8 * stdpdes[i]), pde_loss_function(θ)) - for (i, pde_loss_function) in enumerate(pde_loss_functions[i])] + # first vector of losses,from tuple -> pde losses, first[1] pde loss + pde_loglikelihoods = [[logpdf(Normal(0, stdpdes[j]), pde_loss_function(θ)) + for (j, pde_loss_function) in enumerate(pde_loss_functions[i][1])] for i in eachindex(pde_loss_functions)] # bc_loglikelihoods = [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) # for (j, bc_loss_function) in enumerate(bc_loss_functions)] - # println("bc_loglikelihoods : ", bc_loglikelihoods) + return sum(sum(pde_loglikelihoods)) # sum(sum(pde_loglikelihoods) + sum(bc_loglikelihoods)) end diff --git a/src/discretize.jl b/src/discretize.jl index af035980b3..a6c7c3bed1 100644 --- a/src/discretize.jl +++ b/src/discretize.jl @@ -608,7 +608,9 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, function get_likelihood_estimate_function(discretization::BayesianPINN) dataset_pde, dataset_bc = discretization.dataset - + dataset_pde = dataset_pde isa Nothing ? dataset_pde : get_dataset_train_points(eqs, dataset_pde, pinnrep) + dataset_bc = dataset_bc isa Nothing ? dataset_bc : get_dataset_train_points(eqs, dataset_bc, pinnrep) + # required as Physics loss also needed on the discrete dataset domain points # data points are discrete and so by default GridTraining loss applies # passing placeholder dx with GridTraining, it uses data points irl @@ -616,7 +618,9 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, merge_strategy_with_loglikelihood_function(pinnrep, GridTraining(0.1), datafree_pde_loss_functions, - datafree_bc_loss_functions, train_sets_pde = dataset_pde, train_sets_bc = dataset_bc) + datafree_bc_loss_functions, + train_sets_pde = dataset_pde, + train_sets_bc = dataset_bc) else (nothing, nothing) end diff --git a/src/training_strategies.jl b/src/training_strategies.jl index a2e4624f23..ffec6fe2d2 100644 --- a/src/training_strategies.jl +++ b/src/training_strategies.jl @@ -21,15 +21,22 @@ function get_dataset_train_points(eqs, train_sets, pinnrep) dict_indvars = pinnrep.dict_indvars symbols_input = [(i, dict_depvar_input[i]) for i in depvars] + # [(:u, [:t])] eq_args = NeuralPDE.get_argument(eqs, dict_indvars, dict_depvars) + # [[:t]] + points = [] for eq_arg in eq_args a = [] + # for each (depvar,[indvar1..]) if indvari==indvar (eq_arg) for i in eachindex(symbols_input) if symbols_input[i][2] == eq_arg + # include domain points of that depvar + # each loss equation take domain matrix [points..;points..] push!(a, train_sets[i][:, 2:end]') end end + # vcat as new row for next equation push!(points, vcat(a...)) end return points @@ -39,19 +46,16 @@ end function merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, strategy::GridTraining, datafree_pde_loss_function, - datafree_bc_loss_function; train_sets_pde = nothing,train_sets_bc=nothing) + datafree_bc_loss_function; train_sets_pde = nothing, train_sets_bc=nothing) @unpack domains, eqs, bcs, dict_indvars, dict_depvars, flat_init_params = pinnrep eltypeθ = eltype(pinnrep.flat_init_params) # is vec as later each _set in pde_train_sets are columns as points transformed to vector of points (pde_train_sets must be rowwise) pde_loss_functions = if !(train_sets_pde isa Nothing) - - pde_train_sets = get_dataset_train_points(eqs, train_sets_pde, pinnrep) - println(" pde train set : ", pde_train_sets) - println("type pde train set : ", size(pde_train_sets)) pde_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), - pde_train_sets) + train_sets_pde) + [get_loss_function(_loss, _set, eltypeθ, strategy) for (_loss, _set) in zip(datafree_pde_loss_function, pde_train_sets)] @@ -60,12 +64,9 @@ function merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, end bc_loss_functions = if !(train_sets_bc isa Nothing) - - bcs_train_sets = get_dataset_train_points(bcs, train_sets_bc, pinnrep) - println("bcs train set : ", bcs_train_sets) - println("type bcs train set : ", size(bcs_train_sets)) bcs_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), - bcs_train_sets) + train_sets_bc) + [get_loss_function(_loss, _set, eltypeθ, strategy) for (_loss, _set) in zip(datafree_bc_loss_function, bcs_train_sets)] else @@ -92,16 +93,13 @@ function merge_strategy_with_loss_function(pinnrep::PINNRepresentation, pde_train_sets) bcs_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), bcs_train_sets) + pde_loss_functions = [get_loss_function(_loss, _set, eltypeθ, strategy) for (_loss, _set) in zip(datafree_pde_loss_function, pde_train_sets)] - bc_loss_functions = [get_loss_function(_loss, _set, eltypeθ, strategy) - for (_loss, _set) in zip(datafree_bc_loss_function, bcs_train_sets)] - println("pde_train_sets : ",pde_train_sets) - println("pde_train_sets : ",size(pde_train_sets)) - println("bc_train_sets : ",bcs_train_sets) - println("bc_train_sets : ",size(bcs_train_sets)) + for (_loss, _set) in zip(datafree_bc_loss_function, + bcs_train_sets)] pde_loss_functions, bc_loss_functions end diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 0a23cfa669..c659874afa 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -154,159 +154,6 @@ end # @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] end -@parameters t, x, p -@variables u(..) -Dt = Differential(t) -Dx = Differential(x) -eqs = [u(t, x) * Dt(u(t, x)) - cos(p * t) ~ 0, u(t, x) + Dx(u(t, x)) ~ 0.0] -bcs = [u(0, x) ~ 0.0, u(t, 10) ~ 1.0] -domains = [t ∈ Interval(0.0, 2.0), x ∈ Interval(0.0, 2.0)] - -chainl = Lux.Chain(Lux.Dense(2, 6, tanh), Lux.Dense(6, 1)) -initl, st = Lux.setup(Random.default_rng(), chainl) - -@named pde_system = PDESystem(eqs, - bcs, - domains, - [t, x], - [u(t, x)], - [p], - defaults = Dict([p => 4.0])) - -analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) -timepoints = collect(0.0:(1 / 100.0):2.0) -u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] -u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) -dataset = [hcat(u1, u1, timepoints)] - -# checking all training strategies -# discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, -# dataset = [dataset, nothing]) - -discretization = BayesianPINN([chainl], - GridTraining([0.2, 0.2]), - param_estim = true, dataset = [dataset, nothing]) - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05, 0.05], - phystd = [0.01, 0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0, 1 / 20.0], - param = [Normal(3.0, 0.5)], progress = true) - -param = 2 * π -ts = vec(sol1.timepoints[1]) -u_real = [analytic_sol_func1(0.0, t) for t in ts] -u_predict = pmean(sol1.ensemblesol[1]) - -@test u_predict≈u_real atol=1.5 -@test mean(u_predict .- u_real) < 0.1 -@test sol1.estimated_de_params[1]≈param atol=param * 0.3 - -# for bc case, [bc]/bc eqs must be passed along with dataset_bc[i] -# and final loss for bc must be together in a vector(bcs has seperate type of dataset_bc) -# eqs is vector of pde eqs and dataset here is dataset_pde -# normally you get vector of losses -function get_loss_2(pinnrep, dataset, eqs) - depvars = pinnrep.depvars # order is same as dataset and interps - dict_depvar_input = pinnrep.dict_depvar_input - - to_subs, tobe_subs = get_symbols(dict_depvar_input, dataset, depvars) - interp_subs_dict = Dict(tobe_subs[depvar] => to_subs[depvar] for depvar in depvars) - - Dict_differentials = Dict() - exp = toexpr(eqs) - void_value = [recur_expression(exp_i, Dict_differentials) for exp_i in exp] - # Dict_differentials is now filled with Differential operator => diff_i key-value pairs - - # masking operation - a = substitute.(eqs, Ref(Dict_differentials)) - println(a) - b = substitute.(a, Ref(interp_subs_dict)) - println(b) - # reverse dict for re-substituing values of Differential(t)(u(t)) etc - rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) - eqs = substitute.(b, Ref(rev_Dict_differentials)) - # get losses - loss_functions = [NeuralPDE.build_loss_function(pinnrep, - eqs[i], - pinnrep.pde_indvars[i]) for i in eachindex(eqs)] -end - -# >why not mask differential -function get_lossy(pinnrep, dataset, eqs) - depvars = pinnrep.depvars # order is same as dataset and interps - dict_depvar_input = pinnrep.dict_depvar_input - - Dict_differentials = Dict() - exp = toexpr(eqs) - for exp_i in exp - recur_expression(exp_i, Dict_differentials) - end - # Dict_differentials is now filled with Differential operator => diff_i key-value pairs - - # masking operation - println("Dict_differentials : ", Dict_differentials) - a = substitute.(eqs, Ref(Dict_differentials)) - println("Masked Differential term : ", a) - - to_subs, tobe_subs = get_symbols(dict_depvar_input, dataset, depvars, eqs) - # for each row in dataset create u values for substituing in equation, n_equations=n_rows - eq_subs = [Dict(tobe_subs[depvar] => to_subs[depvar][i] for depvar in depvars) - for i in 1:size(dataset[1][:, 1])[1]] - - b = [] - for eq_sub in eq_subs - push!(b, [substitute(a_i, eq_sub) for a_i in a]) - end - - # reverse dict for re-substituing values of Differential(t)(u(t)) etc - rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) - - c = [] - for b_i in b - push!(c, substitute.(b_i, Ref(rev_Dict_differentials))) - end - println("After re Substituting depvars : ", c[1]) - # c = vcat(c...) - println(c) - c - # get losses - # loss_functions = [NeuralPDE.build_loss_function(pinnrep, - # c[i, :][j], - # pinnrep.pde_indvars[j]) for j in eachindex(pinnrep.pde_indvars)] - # return loss_functions -end - -# dataset_pde has normal matrix format -# dataset_bc has format of Vector{typeof(dataset_pde )} as each bc has different domain requirements -function get_symbols(dict_depvar_input, dataset, depvars, eqs) - depvar_vals = [dataset_i[:, 1] for dataset_i in dataset] - # order of depvars - to_subs = Dict(pinnrep.depvars .=> depvar_vals) - - asrt = Symbolics.get_variables.(eqs) - # want only symbols of depvars - temp = unique(reduce(vcat, asrt)) - # now we have all the depvars, we now need all depvars whcih can be substituted with data interps - - tobe_subs = Dict() - for a in depvars - for i in temp - expr = toexpr(i) - if (expr isa Expr) && (expr.args[1] == a) - tobe_subs[a] = i - end - end - end - - return to_subs, tobe_subs -end - -yuh = get_symbols(pinnrep.dict_depvar_input, dataset, pinnrep.depvars, pinnrep.eqs) - function recur_expression(exp, Dict_differentials) for in_exp in exp.args if !(in_exp isa Expr) @@ -318,95 +165,134 @@ function recur_expression(exp, Dict_differentials) # Dict_differentials for masking differential terms # and resubstituting differentials in equations after putting in interpolations # temp = in_exp.args[end] - # in_exp.args[end] = Symbolics.variable(in_exp.args[end]) - - Dict_differentials[in_exp] = Symbolics.variable("diff_$(length(Dict_differentials)+1)") + Dict_differentials[eval(in_exp)] = Symbolics.variable("diff_$(length(Dict_differentials) + 1)") return else recur_expression(in_exp, Dict_differentials) end end end -vars = Symbolics.variable.(hcat(pinnrep.indvars, pinnrep.depvars)) -toexpr(Differential(t)(Differential(u)(u(t))) + u(t) ~ 0).args[2] -eqs -# Differential(t)(u(t)) - cos(p * t) ~ 0 -exprs = toexpr(eqs) -pop = Dict() -recur_expression(exprs, pop) -pop1 = Dict() -for (a, b) in pop - pop1[eval(a)] = b + +@testset "Example 3: 2D Periodic System with New parameter estimation" begin + # Cos(pi*t) periodic curve + @parameters t, p + @variables u(..) + + Dt = Differential(t) + eqs = Dt(u(t)) - cos(p * t) ~ 0 + bcs = [u(0) ~ 0.0] + domains = [t ∈ Interval(0.0, 2.0)] + + chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) + initl, st = Lux.setup(Random.default_rng(), chainl) + + @named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [u(t)], + [p], + defaults = Dict([p => 4.0])) + + analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) + timepoints = collect(0.0:(1 / 100.0):2.0) + u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] + u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) + dataset = [hcat(u1, timepoints)] + + discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, + dataset = [dataset, nothing]) + + # creating dictionary for masking equations + eqs = pde_system.eqs + Dict_differentials = Dict() + exps = toexpr.(eqs) + nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + + sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(6.0, 0.5)], + Dict_differentials = Dict_differentials) + + param = 2 * π + ts = vec(sol1.timepoints[1]) + u_real = [analytic_sol_func1(0.0, t) for t in ts] + u_predict = pmean(sol1.ensemblesol[1]) + + @test u_predict≈u_real atol=1.5 + @test mean(u_predict .- u_real) < 0.1 + @test sol1.estimated_de_params[1]≈param atol=param * 0.3 end -pop1 -a = substitute(eqs, pop1) - -Symbolics.get_variables(eqs[1]) -# eqs=a - -NeuralPDE.get_variables(pinnrep.eqs, pinnrep.dict_indvars, pinnrep.dict_depvars) -NeuralPDE.get_argument(pinnrep.bcs, pinnrep.dict_indvars, pinnrep.dict_depvars) -dx = pinnrep.strategy.dx -eltypeθ = eltype(pinnrep.flat_init_params) - -# solve dataset physics loss for heterogenous case -# create number of equations as number of interpolation and points(n rows) -# follow masking and finally feed training sets as set in interpolations input of u(t,x,..) - -# logic for recursion formula to parse differentials -# # this below has the whole differential term -toexpr(pde_system.eqs[1]).args[2].args[3].args[3] -# toexpr(pde_system.eqs[1]).args[2].args[3].args[3] -# # .args[1] isa ModelingToolkit.Differential - -# logic for interpolation and indvars splatting to get Equation parsing terms -# splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] -# # splat datasets onto Linear interpolations tables -# interps = [LinearInterpolation(splat_i...) for splat_i in splat_form] -# interps = Dict(depvars .=> interps) -# get datasets into splattable form -# splat_form = [[dataset_i[:, i] for i in 1:size(dataset_i)[2]] for dataset_i in dataset] -# # splat datasets onto Linear interpolations tables -# yu = [LinearInterpolation(splat_i...) for splat_i in splat_form] -# Symbol(:($(yu[1]))) - -# logic to contrauct dict to feed for masking -# Dict(interps[depvar] => dict_depvar_input[depvar] for depvar in depvars) - -@parameters t, p -@variables u(..) - -Dt = Differential(t) -eqs = Dt(u(t)) - cos(p * t) ~ 0 -bcs = [u(0) ~ 0.0] -domains = [t ∈ Interval(0.0, 2.0)] - -chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) -initl, st = Lux.setup(Random.default_rng(), chainl) - -@named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [u(t)], - [p], - defaults = Dict([p => 4.0])) - -analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) -timepoints = collect(0.0:(1 / 100.0):2.0) -u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] -u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) -dataset = [hcat(u1, timepoints)] - -# checking all training strategies -discretization = BayesianPINN([chainl], GridTraining(0.01), param_estim = true, - dataset = [dataset, nothing]) - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)], progress = true) \ No newline at end of file + +@testset "Example 4: Lorenz System with New parameter estimation" begin + @parameters t, σ_ + @variables x(..), y(..), z(..) + Dt = Differential(t) + eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), + Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), + Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] + + bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] + domains = [t ∈ Interval(0.0, 1.0)] + + input_ = length(domains) + n = 7 + chain = [ + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + ] + + #Generate Data + function lorenz!(du, u, p, t) + du[1] = 10.0 * (u[2] - u[1]) + du[2] = u[1] * (28.0 - u[3]) - u[2] + du[3] = u[1] * u[2] - (8 / 3) * u[3] + end + + u0 = [1.0; 0.0; 0.0] + tspan = (0.0, 1.0) + prob = ODEProblem(lorenz!, u0, tspan) + sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) + ts = sol.t + us = hcat(sol.u...) + us = us .+ ((0.05 .* randn(size(us))) .* us) + ts_ = hcat(sol(ts).t...)[1, :] + dataset = [hcat(us[i, :], ts_) for i in 1:3] + + discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, + dataset = [dataset, nothing]) + + @named pde_system = PDESystem(eqs, bcs, domains, + [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) + + # creating dictionary for masking equations + eqs = pde_system.eqs + Dict_differentials = Dict() + exps = toexpr.(eqs) + nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + + sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 50, + bcstd = [0.3, 0.3, 0.3], + phystd = [0.1, 0.1, 0.1], + l2std = [1, 1, 1], + priorsNNw = (0.0, 1.0), + saveats = [0.01], + param = [Normal(12.0, 2)], + Dict_differentials = Dict_differentials) + + idealp = 10.0 + p_ = sol1.estimated_de_params[1] + @test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] + # @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] +end \ No newline at end of file From 2fbe4a9b4765542a76d2e48e31f74525b815a922 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 15 Feb 2024 03:27:10 +0530 Subject: [PATCH 20/53] update BPINN_PDEinvsol_tests.jl --- test/BPINN_PDEinvsol_tests.jl | 292 +++++++++++++++++----------------- 1 file changed, 146 insertions(+), 146 deletions(-) diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index c659874afa..de26c0a208 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -7,152 +7,152 @@ using ComponentArrays, ModelingToolkit Random.seed!(100) -@testset "Example 1: 2D Periodic System with parameter estimation" begin - # Cos(pi*t) periodic curve - @parameters t, p - @variables u(..) - - Dt = Differential(t) - eqs = Dt(u(t)) - cos(p * t) ~ 0 - bcs = [u(0) ~ 0.0] - domains = [t ∈ Interval(0.0, 2.0)] - - chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) - initl, st = Lux.setup(Random.default_rng(), chainl) - - @named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [u(t)], - [p], - defaults = Dict([p => 4.0])) - - analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) - timepoints = collect(0.0:(1 / 100.0):2.0) - u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] - u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) - dataset = [hcat(u1, timepoints)] - - # checking all training strategies - discretization = BayesianPINN([chainl], StochasticTraining(200), param_estim = true, - dataset = [dataset, nothing]) - - ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)]) - - discretization = BayesianPINN([chainl], QuasiRandomTraining(200), param_estim = true, - dataset = [dataset, nothing]) - - ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)]) - - discretization = BayesianPINN([chainl], QuadratureTraining(), param_estim = true, - dataset = [dataset, nothing]) - - ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)]) - - discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, - dataset = [dataset, nothing]) - - sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)]) - - param = 2 * π - ts = vec(sol1.timepoints[1]) - u_real = [analytic_sol_func1(0.0, t) for t in ts] - u_predict = pmean(sol1.ensemblesol[1]) - - @test u_predict≈u_real atol=1.5 - @test mean(u_predict .- u_real) < 0.1 - @test sol1.estimated_de_params[1]≈param atol=param * 0.3 -end - -@testset "Example 2: Lorenz System with parameter estimation" begin - @parameters t, σ_ - @variables x(..), y(..), z(..) - Dt = Differential(t) - eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), - Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), - Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] - - bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] - domains = [t ∈ Interval(0.0, 1.0)] - - input_ = length(domains) - n = 7 - chain = [ - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - ] - - #Generate Data - function lorenz!(du, u, p, t) - du[1] = 10.0 * (u[2] - u[1]) - du[2] = u[1] * (28.0 - u[3]) - u[2] - du[3] = u[1] * u[2] - (8 / 3) * u[3] - end - - u0 = [1.0; 0.0; 0.0] - tspan = (0.0, 1.0) - prob = ODEProblem(lorenz!, u0, tspan) - sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) - ts = sol.t - us = hcat(sol.u...) - us = us .+ ((0.05 .* randn(size(us))) .* us) - ts_ = hcat(sol(ts).t...)[1, :] - dataset = [hcat(us[i, :], ts_) for i in 1:3] - - discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, - dataset = [dataset, nothing]) - - @named pde_system = PDESystem(eqs, bcs, domains, - [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) - - sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 50, - bcstd = [0.3, 0.3, 0.3], - phystd = [0.1, 0.1, 0.1], - l2std = [1, 1, 1], - priorsNNw = (0.0, 1.0), - saveats = [0.01], - param = [Normal(12.0, 2)]) - - idealp = 10.0 - p_ = sol1.estimated_de_params[1] - @test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] - # @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] -end +# @testset "Example 1: 2D Periodic System with parameter estimation" begin +# # Cos(pi*t) periodic curve +# @parameters t, p +# @variables u(..) + +# Dt = Differential(t) +# eqs = Dt(u(t)) - cos(p * t) ~ 0 +# bcs = [u(0) ~ 0.0] +# domains = [t ∈ Interval(0.0, 2.0)] + +# chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) +# initl, st = Lux.setup(Random.default_rng(), chainl) + +# @named pde_system = PDESystem(eqs, +# bcs, +# domains, +# [t], +# [u(t)], +# [p], +# defaults = Dict([p => 4.0])) + +# analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) +# timepoints = collect(0.0:(1 / 100.0):2.0) +# u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] +# u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) +# dataset = [hcat(u1, timepoints)] + +# # checking all training strategies +# discretization = BayesianPINN([chainl], StochasticTraining(200), param_estim = true, +# dataset = [dataset, nothing]) + +# ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 1500, +# bcstd = [0.05], +# phystd = [0.01], l2std = [0.01], +# priorsNNw = (0.0, 1.0), +# saveats = [1 / 50.0], +# param = [LogNormal(6.0, 0.5)]) + +# discretization = BayesianPINN([chainl], QuasiRandomTraining(200), param_estim = true, +# dataset = [dataset, nothing]) + +# ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 1500, +# bcstd = [0.05], +# phystd = [0.01], l2std = [0.01], +# priorsNNw = (0.0, 1.0), +# saveats = [1 / 50.0], +# param = [LogNormal(6.0, 0.5)]) + +# discretization = BayesianPINN([chainl], QuadratureTraining(), param_estim = true, +# dataset = [dataset, nothing]) + +# ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 1500, +# bcstd = [0.05], +# phystd = [0.01], l2std = [0.01], +# priorsNNw = (0.0, 1.0), +# saveats = [1 / 50.0], +# param = [LogNormal(6.0, 0.5)]) + +# discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, +# dataset = [dataset, nothing]) + +# sol1 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 1500, +# bcstd = [0.05], +# phystd = [0.01], l2std = [0.01], +# priorsNNw = (0.0, 1.0), +# saveats = [1 / 50.0], +# param = [LogNormal(6.0, 0.5)]) + +# param = 2 * π +# ts = vec(sol1.timepoints[1]) +# u_real = [analytic_sol_func1(0.0, t) for t in ts] +# u_predict = pmean(sol1.ensemblesol[1]) + +# @test u_predict≈u_real atol=1.5 +# @test mean(u_predict .- u_real) < 0.1 +# @test sol1.estimated_de_params[1]≈param atol=param * 0.3 +# end + +# @testset "Example 2: Lorenz System with parameter estimation" begin +# @parameters t, σ_ +# @variables x(..), y(..), z(..) +# Dt = Differential(t) +# eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), +# Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), +# Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] + +# bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] +# domains = [t ∈ Interval(0.0, 1.0)] + +# input_ = length(domains) +# n = 7 +# chain = [ +# Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), +# Lux.Dense(n, 1)), +# Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), +# Lux.Dense(n, 1)), +# Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), +# Lux.Dense(n, 1)), +# ] + +# #Generate Data +# function lorenz!(du, u, p, t) +# du[1] = 10.0 * (u[2] - u[1]) +# du[2] = u[1] * (28.0 - u[3]) - u[2] +# du[3] = u[1] * u[2] - (8 / 3) * u[3] +# end + +# u0 = [1.0; 0.0; 0.0] +# tspan = (0.0, 1.0) +# prob = ODEProblem(lorenz!, u0, tspan) +# sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) +# ts = sol.t +# us = hcat(sol.u...) +# us = us .+ ((0.05 .* randn(size(us))) .* us) +# ts_ = hcat(sol(ts).t...)[1, :] +# dataset = [hcat(us[i, :], ts_) for i in 1:3] + +# discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, +# dataset = [dataset, nothing]) + +# @named pde_system = PDESystem(eqs, bcs, domains, +# [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) + +# sol1 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 50, +# bcstd = [0.3, 0.3, 0.3], +# phystd = [0.1, 0.1, 0.1], +# l2std = [1, 1, 1], +# priorsNNw = (0.0, 1.0), +# saveats = [0.01], +# param = [Normal(12.0, 2)]) + +# idealp = 10.0 +# p_ = sol1.estimated_de_params[1] +# @test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] +# # @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] +# end function recur_expression(exp, Dict_differentials) for in_exp in exp.args From e0028028f63fa1a343e05bf8021c9f001b8130ba Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 15 Feb 2024 10:42:52 +0530 Subject: [PATCH 21/53] spellings, newloss now optional --- src/PDE_BPINN.jl | 74 +++++---- src/collocated_estim.jl | 2 +- test/BPINN_PDE_tests.jl | 2 +- test/BPINN_PDEinvsol_tests.jl | 292 +++++++++++++++++----------------- 4 files changed, 188 insertions(+), 182 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index 7cbe22a804..a223c58e20 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -64,7 +64,7 @@ mutable struct PDELogTargetDensity{ end # for bc case, [bc]/bc eqs must be passed along with dataset_bc[i] -# and final loss for bc must be together in a vector(bcs has seperate type of dataset_bc) +# and final loss for bc must be together in a vector(bcs has separate type of dataset_bc) # eqs is vector of pde eqs and dataset here is dataset_pde # normally you get vector of losses function get_lossy(pinnrep, dataset, Dict_differentials) @@ -76,7 +76,7 @@ function get_lossy(pinnrep, dataset, Dict_differentials) eqs_new = substitute.(eqs, Ref(Dict_differentials)) to_subs, tobe_subs = get_symbols(dataset, depvars, eqs) - # for each row in dataset create u values for substituing in equation, n_equations=n_rows + # for each row in dataset create u values for substituting in equation, n_equations=n_rows eq_subs = [Dict(tobe_subs[depvar] => to_subs[depvar][i] for depvar in depvars) for i in 1:size(dataset[1][:, 1])[1]] @@ -87,10 +87,10 @@ function get_lossy(pinnrep, dataset, Dict_differentials) end # now we have vector of equation vectors - # reverse dict for re-substituing values of Differential(t)(u(t)) etc + # reverse dict for re-substituting values of Differential(t)(u(t)) etc rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) - # for each vector in vecvtor of equation vectorbroadcast resubstituing OG mask values + # for each vector in vector of equation vectorbroadcast resubstituing OG mask values c = [] for b_i in b push!(c, substitute.(b_i, Ref(rev_Dict_differentials))) @@ -131,9 +131,11 @@ end function LogDensityProblems.logdensity(Tar::PDELogTargetDensity, θ) # for parameter estimation neccesarry to use multioutput case - return Tar.full_loglikelihood(setparameters(Tar, θ), - Tar.allstd) + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) + - Tar.L2_loss2(setparameters(Tar, θ), Tar.allstd) + if Tar.L2_loss2 isa Nothing + return Tar.full_loglikelihood(setparameters(Tar, θ),Tar.allstd) + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) + else + return Tar.full_loglikelihood(setparameters(Tar, θ), Tar.allstd) + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) + Tar.L2_loss2(setparameters(Tar, θ), Tar.allstd) + end end function setparameters(Tar::PDELogTargetDensity, θ) @@ -360,38 +362,42 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; Adaptorkwargs = (Adaptor = StanHMCAdaptor, Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), Integratorkwargs = (Integrator = Leapfrog,), saveats = [1 / 10.0], - numensemble = floor(Int, draw_samples / 3), Dict_differentials = Dict(), + numensemble = floor(Int, draw_samples / 3), Dict_differentials = nothing, progress = false, verbose = false) pinnrep = symbolic_discretize(pde_system, discretization) dataset_pde, dataset_bc = discretization.dataset - yuh1 = get_lossy(pinnrep, dataset_pde, Dict_differentials) - # eqs = pinnrep.bcs - # yuh2 = get_lossy(pinnrep, dataset_pde, eqs) - - # this is a vector of tuple{vector,nothing} - pde_loss_functions = [merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, - GridTraining(0.1), - yuh1[i], - nothing; train_sets_pde = [data_pde[i, :] for data_pde in dataset_pde], - train_sets_bc = nothing) - for i in eachindex(yuh1)] - - function L2_loss2(θ, allstd) - stdpdes, stdbcs, stdextra = allstd - # first vector of losses,from tuple -> pde losses, first[1] pde loss - pde_loglikelihoods = [[logpdf(Normal(0, stdpdes[j]), pde_loss_function(θ)) - for (j, pde_loss_function) in enumerate(pde_loss_functions[i][1])] - for i in eachindex(pde_loss_functions)] - - # bc_loglikelihoods = [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) - # for (j, bc_loss_function) in enumerate(bc_loss_functions)] - - return sum(sum(pde_loglikelihoods)) - # sum(sum(pde_loglikelihoods) + sum(bc_loglikelihoods)) + newloss = if Dict_differentials isa Nothing + nothing + else + yuh1 = get_lossy(pinnrep, dataset_pde, Dict_differentials) + # eqs = pinnrep.bcs + # yuh2 = get_lossy(pinnrep, dataset_pde, eqs) + + # this is a vector of tuple{vector,nothing} + pde_loss_functions = [merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, + GridTraining(0.1), + yuh1[i], + nothing; train_sets_pde = [data_pde[i, :] for data_pde in dataset_pde], + train_sets_bc = nothing) + for i in eachindex(yuh1)] + + function L2_loss2(θ, allstd) + stdpdes, stdbcs, stdextra = allstd + # first vector of losses,from tuple -> pde losses, first[1] pde loss + pde_loglikelihoods = [[logpdf(Normal(0, stdpdes[j]), pde_loss_function(θ)) + for (j, pde_loss_function) in enumerate(pde_loss_functions[i][1])] + for i in eachindex(pde_loss_functions)] + + # bc_loglikelihoods = [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) + # for (j, bc_loss_function) in enumerate(bc_loss_functions)] + + return sum(sum(pde_loglikelihoods)) + # sum(sum(pde_loglikelihoods) + sum(bc_loglikelihoods)) + end end - # WIP split dataset to respective equations + # [WIP] add overall functionality for BC dataset points if ((dataset_bc isa Nothing) && (dataset_pde isa Nothing)) dataset = nothing elseif dataset_bc isa Nothing @@ -468,7 +474,7 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; ninv, initial_nnθ, full_weighted_loglikelihood, - L2_loss2, + newloss, Φ) Adaptor, Metric, targetacceptancerate = Adaptorkwargs[:Adaptor], diff --git a/src/collocated_estim.jl b/src/collocated_estim.jl index b113b76f12..a2f81b3ed9 100644 --- a/src/collocated_estim.jl +++ b/src/collocated_estim.jl @@ -175,7 +175,7 @@ function calculate_derivatives(dataset) # # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) # interp = CubicSpline(u, t) # interp1 = CubicSpline(u1, t) - # # derrivatives interpolation + # # derivatives interpolation # dx = t[2] - t[1] # time = collect(t[1]:dx:t[end]) # smoothu = [interp(i) for i in time] diff --git a/test/BPINN_PDE_tests.jl b/test/BPINN_PDE_tests.jl index cd8f6ef466..6dd3637f5a 100644 --- a/test/BPINN_PDE_tests.jl +++ b/test/BPINN_PDE_tests.jl @@ -205,4 +205,4 @@ end u_real = vec([analytic_sol_func(t) for t in ts]) u_predict = pmean(sol1.ensemblesol[1]) @test u_predict≈u_real atol=0.8 -end \ No newline at end of file +end diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index de26c0a208..c659874afa 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -7,152 +7,152 @@ using ComponentArrays, ModelingToolkit Random.seed!(100) -# @testset "Example 1: 2D Periodic System with parameter estimation" begin -# # Cos(pi*t) periodic curve -# @parameters t, p -# @variables u(..) - -# Dt = Differential(t) -# eqs = Dt(u(t)) - cos(p * t) ~ 0 -# bcs = [u(0) ~ 0.0] -# domains = [t ∈ Interval(0.0, 2.0)] - -# chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) -# initl, st = Lux.setup(Random.default_rng(), chainl) - -# @named pde_system = PDESystem(eqs, -# bcs, -# domains, -# [t], -# [u(t)], -# [p], -# defaults = Dict([p => 4.0])) - -# analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) -# timepoints = collect(0.0:(1 / 100.0):2.0) -# u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] -# u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) -# dataset = [hcat(u1, timepoints)] - -# # checking all training strategies -# discretization = BayesianPINN([chainl], StochasticTraining(200), param_estim = true, -# dataset = [dataset, nothing]) - -# ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 1500, -# bcstd = [0.05], -# phystd = [0.01], l2std = [0.01], -# priorsNNw = (0.0, 1.0), -# saveats = [1 / 50.0], -# param = [LogNormal(6.0, 0.5)]) - -# discretization = BayesianPINN([chainl], QuasiRandomTraining(200), param_estim = true, -# dataset = [dataset, nothing]) - -# ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 1500, -# bcstd = [0.05], -# phystd = [0.01], l2std = [0.01], -# priorsNNw = (0.0, 1.0), -# saveats = [1 / 50.0], -# param = [LogNormal(6.0, 0.5)]) - -# discretization = BayesianPINN([chainl], QuadratureTraining(), param_estim = true, -# dataset = [dataset, nothing]) - -# ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 1500, -# bcstd = [0.05], -# phystd = [0.01], l2std = [0.01], -# priorsNNw = (0.0, 1.0), -# saveats = [1 / 50.0], -# param = [LogNormal(6.0, 0.5)]) - -# discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, -# dataset = [dataset, nothing]) - -# sol1 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 1500, -# bcstd = [0.05], -# phystd = [0.01], l2std = [0.01], -# priorsNNw = (0.0, 1.0), -# saveats = [1 / 50.0], -# param = [LogNormal(6.0, 0.5)]) - -# param = 2 * π -# ts = vec(sol1.timepoints[1]) -# u_real = [analytic_sol_func1(0.0, t) for t in ts] -# u_predict = pmean(sol1.ensemblesol[1]) - -# @test u_predict≈u_real atol=1.5 -# @test mean(u_predict .- u_real) < 0.1 -# @test sol1.estimated_de_params[1]≈param atol=param * 0.3 -# end - -# @testset "Example 2: Lorenz System with parameter estimation" begin -# @parameters t, σ_ -# @variables x(..), y(..), z(..) -# Dt = Differential(t) -# eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), -# Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), -# Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] - -# bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] -# domains = [t ∈ Interval(0.0, 1.0)] - -# input_ = length(domains) -# n = 7 -# chain = [ -# Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), -# Lux.Dense(n, 1)), -# Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), -# Lux.Dense(n, 1)), -# Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), -# Lux.Dense(n, 1)), -# ] - -# #Generate Data -# function lorenz!(du, u, p, t) -# du[1] = 10.0 * (u[2] - u[1]) -# du[2] = u[1] * (28.0 - u[3]) - u[2] -# du[3] = u[1] * u[2] - (8 / 3) * u[3] -# end - -# u0 = [1.0; 0.0; 0.0] -# tspan = (0.0, 1.0) -# prob = ODEProblem(lorenz!, u0, tspan) -# sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) -# ts = sol.t -# us = hcat(sol.u...) -# us = us .+ ((0.05 .* randn(size(us))) .* us) -# ts_ = hcat(sol(ts).t...)[1, :] -# dataset = [hcat(us[i, :], ts_) for i in 1:3] - -# discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, -# dataset = [dataset, nothing]) - -# @named pde_system = PDESystem(eqs, bcs, domains, -# [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) - -# sol1 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 50, -# bcstd = [0.3, 0.3, 0.3], -# phystd = [0.1, 0.1, 0.1], -# l2std = [1, 1, 1], -# priorsNNw = (0.0, 1.0), -# saveats = [0.01], -# param = [Normal(12.0, 2)]) - -# idealp = 10.0 -# p_ = sol1.estimated_de_params[1] -# @test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] -# # @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] -# end +@testset "Example 1: 2D Periodic System with parameter estimation" begin + # Cos(pi*t) periodic curve + @parameters t, p + @variables u(..) + + Dt = Differential(t) + eqs = Dt(u(t)) - cos(p * t) ~ 0 + bcs = [u(0) ~ 0.0] + domains = [t ∈ Interval(0.0, 2.0)] + + chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) + initl, st = Lux.setup(Random.default_rng(), chainl) + + @named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [u(t)], + [p], + defaults = Dict([p => 4.0])) + + analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) + timepoints = collect(0.0:(1 / 100.0):2.0) + u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] + u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) + dataset = [hcat(u1, timepoints)] + + # checking all training strategies + discretization = BayesianPINN([chainl], StochasticTraining(200), param_estim = true, + dataset = [dataset, nothing]) + + ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(6.0, 0.5)]) + + discretization = BayesianPINN([chainl], QuasiRandomTraining(200), param_estim = true, + dataset = [dataset, nothing]) + + ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(6.0, 0.5)]) + + discretization = BayesianPINN([chainl], QuadratureTraining(), param_estim = true, + dataset = [dataset, nothing]) + + ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(6.0, 0.5)]) + + discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, + dataset = [dataset, nothing]) + + sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(6.0, 0.5)]) + + param = 2 * π + ts = vec(sol1.timepoints[1]) + u_real = [analytic_sol_func1(0.0, t) for t in ts] + u_predict = pmean(sol1.ensemblesol[1]) + + @test u_predict≈u_real atol=1.5 + @test mean(u_predict .- u_real) < 0.1 + @test sol1.estimated_de_params[1]≈param atol=param * 0.3 +end + +@testset "Example 2: Lorenz System with parameter estimation" begin + @parameters t, σ_ + @variables x(..), y(..), z(..) + Dt = Differential(t) + eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), + Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), + Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] + + bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] + domains = [t ∈ Interval(0.0, 1.0)] + + input_ = length(domains) + n = 7 + chain = [ + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + ] + + #Generate Data + function lorenz!(du, u, p, t) + du[1] = 10.0 * (u[2] - u[1]) + du[2] = u[1] * (28.0 - u[3]) - u[2] + du[3] = u[1] * u[2] - (8 / 3) * u[3] + end + + u0 = [1.0; 0.0; 0.0] + tspan = (0.0, 1.0) + prob = ODEProblem(lorenz!, u0, tspan) + sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) + ts = sol.t + us = hcat(sol.u...) + us = us .+ ((0.05 .* randn(size(us))) .* us) + ts_ = hcat(sol(ts).t...)[1, :] + dataset = [hcat(us[i, :], ts_) for i in 1:3] + + discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, + dataset = [dataset, nothing]) + + @named pde_system = PDESystem(eqs, bcs, domains, + [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) + + sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 50, + bcstd = [0.3, 0.3, 0.3], + phystd = [0.1, 0.1, 0.1], + l2std = [1, 1, 1], + priorsNNw = (0.0, 1.0), + saveats = [0.01], + param = [Normal(12.0, 2)]) + + idealp = 10.0 + p_ = sol1.estimated_de_params[1] + @test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] + # @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] +end function recur_expression(exp, Dict_differentials) for in_exp in exp.args From b26a75bc4aacff6ec6a38de04d5eb04c3b902cb9 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 15 Feb 2024 12:44:09 +0530 Subject: [PATCH 22/53] update PDE_BPINN.jl --- src/PDE_BPINN.jl | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index a223c58e20..ec17257ef3 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -132,9 +132,12 @@ end function LogDensityProblems.logdensity(Tar::PDELogTargetDensity, θ) # for parameter estimation neccesarry to use multioutput case if Tar.L2_loss2 isa Nothing - return Tar.full_loglikelihood(setparameters(Tar, θ),Tar.allstd) + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) + return Tar.full_loglikelihood(setparameters(Tar, θ), Tar.allstd) + + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) else - return Tar.full_loglikelihood(setparameters(Tar, θ), Tar.allstd) + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) + Tar.L2_loss2(setparameters(Tar, θ), Tar.allstd) + return Tar.full_loglikelihood(setparameters(Tar, θ), Tar.allstd) + + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) + + Tar.L2_loss2(setparameters(Tar, θ), Tar.allstd) end end @@ -489,9 +492,11 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; ℓπ.allstd)) @info("Current Prior Log-likelihood : ", priorlogpdf(ℓπ, initial_θ)) @info("Current MSE against dataset Log-likelihood : ", L2LossData(ℓπ, initial_θ)) - @info("Current L2_LOSSY : ", - ℓπ.L2_loss2(setparameters(ℓπ, initial_θ), - ℓπ.allstd)) + if !(newloss isa Nothing) + @info("Current L2_LOSSY : ", + ℓπ.L2_loss2(setparameters(ℓπ, initial_θ), + ℓπ.allstd)) + end # parallel sampling option if nchains != 1 @@ -550,9 +555,11 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; @info("Current Prior Log-likelihood : ", priorlogpdf(ℓπ, samples[end])) @info("Current MSE against dataset Log-likelihood : ", L2LossData(ℓπ, samples[end])) - @info("Current L2_LOSSY : ", - ℓπ.L2_loss2(setparameters(ℓπ, samples[end]), - ℓπ.allstd)) + if !(newloss isa Nothing) + @info("Current L2_LOSSY : ", + ℓπ.L2_loss2(setparameters(ℓπ, samples[end]), + ℓπ.allstd)) + end fullsolution = BPINNstats(mcmc_chain, samples, stats) ensemblecurves, estimnnparams, estimated_params, timepoints = inference(samples, From f4b1bfb6c3f6b500e795461724a769695026175c Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 15 Feb 2024 15:46:01 +0530 Subject: [PATCH 23/53] removed length reweighing in BPINN ode, testset for recur.. --- src/advancedHMC_MCMC.jl | 18 ++- test/BPINN_PDEinvsol_tests.jl | 239 +++++++++++++++++----------------- 2 files changed, 131 insertions(+), 126 deletions(-) diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index e5797e7924..ea50eabcf6 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -86,10 +86,19 @@ end vector_to_parameters(ps_new::AbstractVector, ps::AbstractVector) = ps_new function LogDensityProblems.logdensity(Tar::LogTargetDensity, θ) + # if Tar.estim_collocate + # return physloglikelihood(Tar, θ) / length(Tar.dataset[1]) + priorweights(Tar, θ) + + # L2LossData(Tar, θ) / length(Tar.dataset[1]) + + # L2loss2(Tar, θ) / length(Tar.dataset[1]) + # else + # return physloglikelihood(Tar, θ) / length(Tar.dataset[1]) + priorweights(Tar, θ) + + # L2LossData(Tar, θ) / length(Tar.dataset[1]) + # end if Tar.estim_collocate - return physloglikelihood(Tar, θ)/length(Tar.dataset[1]) + priorweights(Tar, θ) + L2LossData(Tar, θ)/length(Tar.dataset[1]) + L2loss2(Tar, θ)/length(Tar.dataset[1]) + return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) + + L2loss2(Tar, θ) else - return physloglikelihood(Tar, θ)/length(Tar.dataset[1]) + priorweights(Tar, θ) + L2LossData(Tar, θ)/length(Tar.dataset[1]) + return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) end end @@ -446,7 +455,6 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; MCMCkwargs = (n_leapfrog = 30,), progress = false, verbose = false, estim_collocate = false) - !(chain isa Lux.AbstractExplicitLayer) && (chain = Lux.transform(chain)) # NN parameter prior mean and variance(PriorsNN must be a tuple) if isinplace(prob) @@ -570,9 +578,9 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; @info("Current Prior Log-likelihood : ", priorweights(ℓπ, samples[end])) @info("Current MSE against dataset Log-likelihood : ", L2LossData(ℓπ, samples[end])) - + # return a chain(basic chain),samples and stats - matrix_samples = reshape(hcat(samples...), (length(samples[1]), length(samples), 1)) + matrix_samples = reshape(hcat(samples...), (length(samples[1]), length(samples), 1)) mcmc_chain = MCMCChains.Chains(matrix_samples) return mcmc_chain, samples, stats end diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index c659874afa..32076a0e9e 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -173,126 +173,123 @@ function recur_expression(exp, Dict_differentials) end end -@testset "Example 3: 2D Periodic System with New parameter estimation" begin - # Cos(pi*t) periodic curve - @parameters t, p - @variables u(..) - - Dt = Differential(t) - eqs = Dt(u(t)) - cos(p * t) ~ 0 - bcs = [u(0) ~ 0.0] - domains = [t ∈ Interval(0.0, 2.0)] - - chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) - initl, st = Lux.setup(Random.default_rng(), chainl) - - @named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [u(t)], - [p], - defaults = Dict([p => 4.0])) - - analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) - timepoints = collect(0.0:(1 / 100.0):2.0) - u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] - u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) - dataset = [hcat(u1, timepoints)] - - discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, - dataset = [dataset, nothing]) - - # creating dictionary for masking equations - eqs = pde_system.eqs - Dict_differentials = Dict() - exps = toexpr.(eqs) - nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - - sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)], - Dict_differentials = Dict_differentials) - - param = 2 * π - ts = vec(sol1.timepoints[1]) - u_real = [analytic_sol_func1(0.0, t) for t in ts] - u_predict = pmean(sol1.ensemblesol[1]) - - @test u_predict≈u_real atol=1.5 - @test mean(u_predict .- u_real) < 0.1 - @test sol1.estimated_de_params[1]≈param atol=param * 0.3 +println("Example 3: 2D Periodic System with New parameter estimation") +@parameters t, p +@variables u(..) + +Dt = Differential(t) +eqs = Dt(u(t)) - cos(p * t) ~ 0 +bcs = [u(0) ~ 0.0] +domains = [t ∈ Interval(0.0, 2.0)] + +chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) +initl, st = Lux.setup(Random.default_rng(), chainl) + +@named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [u(t)], + [p], + defaults = Dict([p => 4.0])) + +analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) +timepoints = collect(0.0:(1 / 100.0):2.0) +u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] +u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) +dataset = [hcat(u1, timepoints)] + +discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, + dataset = [dataset, nothing]) + +# creating dictionary for masking equations +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(6.0, 0.5)], + Dict_differentials = Dict_differentials) + +param = 2 * π +ts = vec(sol1.timepoints[1]) +u_real = [analytic_sol_func1(0.0, t) for t in ts] +u_predict = pmean(sol1.ensemblesol[1]) + +@test u_predict≈u_real atol=1.5 +@test mean(u_predict .- u_real) < 0.1 +@test sol1.estimated_de_params[1]≈param atol=param * 0.3 + +println("Example 4: Lorenz System with New parameter estimation") +@parameters t, σ_ +@variables x(..), y(..), z(..) +Dt = Differential(t) +eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), + Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), + Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] + +bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] +domains = [t ∈ Interval(0.0, 1.0)] + +input_ = length(domains) +n = 7 +chain = [ + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), +] + +#Generate Data +function lorenz!(du, u, p, t) + du[1] = 10.0 * (u[2] - u[1]) + du[2] = u[1] * (28.0 - u[3]) - u[2] + du[3] = u[1] * u[2] - (8 / 3) * u[3] end -@testset "Example 4: Lorenz System with New parameter estimation" begin - @parameters t, σ_ - @variables x(..), y(..), z(..) - Dt = Differential(t) - eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), - Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), - Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] - - bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] - domains = [t ∈ Interval(0.0, 1.0)] - - input_ = length(domains) - n = 7 - chain = [ - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - ] - - #Generate Data - function lorenz!(du, u, p, t) - du[1] = 10.0 * (u[2] - u[1]) - du[2] = u[1] * (28.0 - u[3]) - u[2] - du[3] = u[1] * u[2] - (8 / 3) * u[3] - end - - u0 = [1.0; 0.0; 0.0] - tspan = (0.0, 1.0) - prob = ODEProblem(lorenz!, u0, tspan) - sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) - ts = sol.t - us = hcat(sol.u...) - us = us .+ ((0.05 .* randn(size(us))) .* us) - ts_ = hcat(sol(ts).t...)[1, :] - dataset = [hcat(us[i, :], ts_) for i in 1:3] - - discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, - dataset = [dataset, nothing]) - - @named pde_system = PDESystem(eqs, bcs, domains, - [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) - - # creating dictionary for masking equations - eqs = pde_system.eqs - Dict_differentials = Dict() - exps = toexpr.(eqs) - nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - - sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 50, - bcstd = [0.3, 0.3, 0.3], - phystd = [0.1, 0.1, 0.1], - l2std = [1, 1, 1], - priorsNNw = (0.0, 1.0), - saveats = [0.01], - param = [Normal(12.0, 2)], - Dict_differentials = Dict_differentials) - - idealp = 10.0 - p_ = sol1.estimated_de_params[1] - @test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] - # @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] -end \ No newline at end of file +u0 = [1.0; 0.0; 0.0] +tspan = (0.0, 1.0) +prob = ODEProblem(lorenz!, u0, tspan) +sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) +ts = sol.t +us = hcat(sol.u...) +us = us .+ ((0.05 .* randn(size(us))) .* us) +ts_ = hcat(sol(ts).t...)[1, :] +dataset = [hcat(us[i, :], ts_) for i in 1:3] + +discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, + dataset = [dataset, nothing]) + +@named pde_system = PDESystem(eqs, bcs, domains, + [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) + +# creating dictionary for masking equations +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 50, + bcstd = [0.3, 0.3, 0.3], + phystd = [0.1, 0.1, 0.1], + l2std = [1, 1, 1], + priorsNNw = (0.0, 1.0), + saveats = [0.01], + param = [Normal(12.0, 2)], + Dict_differentials = Dict_differentials) + +idealp = 10.0 +p_ = sol1.estimated_de_params[1] +@test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] +# @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] From cd01ceeb67caa3aa832cd4d32c2134c2cb7c8242 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sat, 17 Feb 2024 01:47:58 +0530 Subject: [PATCH 24/53] corrected tests, datasetnew format --- src/PDE_BPINN.jl | 16 +++++++++++----- src/discretize.jl | 2 +- test/BPINN_Tests.jl | 20 ++++++++++---------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index ec17257ef3..69eaa2b733 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -360,7 +360,7 @@ end function ahmc_bayesian_pinn_pde(pde_system, discretization; draw_samples = 1000, bcstd = [0.01], l2std = [0.05], - phystd = [0.05], priorsNNw = (0.0, 2.0), + phystd = [0.05], phystdnew = [0.05], priorsNNw = (0.0, 2.0), param = [], nchains = 1, Kernel = HMC(0.1, 30), Adaptorkwargs = (Adaptor = StanHMCAdaptor, Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), @@ -377,18 +377,24 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; # eqs = pinnrep.bcs # yuh2 = get_lossy(pinnrep, dataset_pde, eqs) + # consider all dataset domain points and for each row new set of equation loss function # this is a vector of tuple{vector,nothing} pde_loss_functions = [merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, GridTraining(0.1), yuh1[i], - nothing; train_sets_pde = [data_pde[i, :] for data_pde in dataset_pde], + nothing; + # pass transformation of each dataset row-corresponds to each point, for each depvar dataset point merged equation vector + train_sets_pde = get_dataset_train_points(pde_system.eqs, + [Array(data[i, :]') for data in dataset_pde], + pinnrep), train_sets_bc = nothing) for i in eachindex(yuh1)] function L2_loss2(θ, allstd) - stdpdes, stdbcs, stdextra = allstd + stdpdesnew = allstd[4] + # first vector of losses,from tuple -> pde losses, first[1] pde loss - pde_loglikelihoods = [[logpdf(Normal(0, stdpdes[j]), pde_loss_function(θ)) + pde_loglikelihoods = [[logpdf(Normal(0, stdpdesnew[j]), pde_loss_function(θ)) for (j, pde_loss_function) in enumerate(pde_loss_functions[i][1])] for i in eachindex(pde_loss_functions)] @@ -472,7 +478,7 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; strategy, dataset, priors, - [phystd, bcstd, l2std], + [phystd, bcstd, l2std, phystdnew], names, ninv, initial_nnθ, diff --git a/src/discretize.jl b/src/discretize.jl index a6c7c3bed1..0740544c09 100644 --- a/src/discretize.jl +++ b/src/discretize.jl @@ -626,7 +626,7 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, end function full_loss_function(θ, allstd::Vector{Vector{Float64}}) - stdpdes, stdbcs, stdextra = allstd + stdpdes, stdbcs, stdextra, stdpdesnew = allstd # the aggregation happens on cpu even if the losses are gpu, probably fine since it's only a few of them pde_loglikelihoods = [logpdf(Normal(0, stdpdes[i]), pde_loss_function(θ)) for (i, pde_loss_function) in enumerate(pde_loss_functions)] diff --git a/test/BPINN_Tests.jl b/test/BPINN_Tests.jl index 6821a8d35e..2fe347b3b4 100644 --- a/test/BPINN_Tests.jl +++ b/test/BPINN_Tests.jl @@ -43,8 +43,8 @@ Random.seed!(100) # testing points t = time # Mean of last 500 sampled parameter's curves[Ensemble predictions] - θ = [vector_to_parameters(fhsamples[i], θinit) for i in 2000:2500] - luxar = [chainlux(t', θ[i], st)[1] for i in 1:500] + θ = [vector_to_parameters(fhsamples[i], θinit) for i in 2000:length(fhsamples)] + luxar = [chainlux(t', θ[i], st)[1] for i in eachindex(θ)] luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] meanscurve = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean @@ -110,8 +110,8 @@ end # testing points t = time # Mean of last 500 sampled parameter's curves(flux and lux chains)[Ensemble predictions] - θ = [vector_to_parameters(fhsamples[i][1:(end - 1)], θinit) for i in 2000:2500] - luxar = [chainlux1(t', θ[i], st)[1] for i in 1:500] + θ = [vector_to_parameters(fhsamples[i][1:(end - 1)], θinit) for i in 2000:length(fhsamples)] + luxar = [chainlux1(t', θ[i], st)[1] for i in eachindex(θ)] luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] meanscurve = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean @@ -119,7 +119,7 @@ end @test mean(abs.(physsol1 .- meanscurve)) < 0.15 # ESTIMATED ODE PARAMETERS (NN1 AND NN2) - @test abs(p - mean([fhsamples[i][23] for i in 2000:2500])) < abs(0.35 * p) + @test abs(p - mean([fhsamples[i][23] for i in 2000:length(fhsamples)])) < abs(0.35 * p) #-------------------------- solve() call @test mean(abs.(physsol1_1 .- sol2lux.ensemblesol[1])) < 8e-2 @@ -190,13 +190,13 @@ end t = sol.t #------------------------------ ahmc_bayesian_pinn_ode() call # Mean of last 500 sampled parameter's curves(lux chains)[Ensemble predictions] - θ = [vector_to_parameters(fhsampleslux12[i], θinit) for i in 1000:1500] - luxar = [chainlux12(t', θ[i], st)[1] for i in 1:500] + θ = [vector_to_parameters(fhsampleslux12[i], θinit) for i in 1000:length(fhsampleslux12)] + luxar = [chainlux12(t', θ[i], st)[1] for i in eachindex(θ)] luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] meanscurve2_1 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean - θ = [vector_to_parameters(fhsampleslux22[i][1:(end - 1)], θinit) for i in 1000:1500] - luxar = [chainlux12(t', θ[i], st)[1] for i in 1:500] + θ = [vector_to_parameters(fhsampleslux22[i][1:(end - 1)], θinit) for i in 1000:length(fhsampleslux22)] + luxar = [chainlux12(t', θ[i], st)[1] for i in eachindex(θ)] luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] meanscurve2_2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean @@ -206,7 +206,7 @@ end @test mean(abs.(physsol1 .- meanscurve2_2)) < 5e-2 # estimated parameters(lux chain) - param1 = mean(i[62] for i in fhsampleslux22[1000:1500]) + param1 = mean(i[62] for i in fhsampleslux22[1000:length(fhsampleslux22)]) @test abs(param1 - p) < abs(0.3 * p) #-------------------------- solve() call From 78cadf1ac9f5d91a0c4378ad500d43ccb089b5b8 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 22 Feb 2024 00:32:26 +0530 Subject: [PATCH 25/53] changes from reviews --- src/PDE_BPINN.jl | 12 +- src/training_strategies.jl | 18 +- test/BPINN_PDEinvsol_tests.jl | 303 ++++++++++++++++++++++++++++------ 3 files changed, 258 insertions(+), 75 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index 69eaa2b733..500510d64c 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -81,20 +81,15 @@ function get_lossy(pinnrep, dataset, Dict_differentials) for i in 1:size(dataset[1][:, 1])[1]] # for each point(eq_sub dictionary) substiute in all equations(eqs_new - masked equations) - b = [] - for eq_sub in eq_subs - push!(b, [substitute(eq, eq_sub) for eq in eqs_new]) - end + b = [[substitute(eq, eq_sub) for eq in eqs_new] for eq_sub in eq_subs] + # now we have vector of equation vectors # reverse dict for re-substituting values of Differential(t)(u(t)) etc rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) # for each vector in vector of equation vectorbroadcast resubstituing OG mask values - c = [] - for b_i in b - push!(c, substitute.(b_i, Ref(rev_Dict_differentials))) - end + c = [substitute.(b_i, Ref(rev_Dict_differentials)) for b_i in b] # get losses, zip each equation with args for each build_loss call per equation vector loss_functions = [[build_loss_function(pinnrep, eq, pde_indvar) @@ -506,7 +501,6 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; # parallel sampling option if nchains != 1 - # Cache to store the chains bpinnsols = Vector{Any}(undef, nchains) diff --git a/src/training_strategies.jl b/src/training_strategies.jl index ffec6fe2d2..1bf767beca 100644 --- a/src/training_strategies.jl +++ b/src/training_strategies.jl @@ -25,20 +25,10 @@ function get_dataset_train_points(eqs, train_sets, pinnrep) eq_args = NeuralPDE.get_argument(eqs, dict_indvars, dict_depvars) # [[:t]] - points = [] - for eq_arg in eq_args - a = [] - # for each (depvar,[indvar1..]) if indvari==indvar (eq_arg) - for i in eachindex(symbols_input) - if symbols_input[i][2] == eq_arg - # include domain points of that depvar - # each loss equation take domain matrix [points..;points..] - push!(a, train_sets[i][:, 2:end]') - end - end - # vcat as new row for next equation - push!(points, vcat(a...)) - end + points = [vcat([train_sets[i][:, 2:end]' + for i in eachindex(symbols_input) if symbols_input[i][2] == eq_arg]...) + for eq_arg in eq_args] + return points end diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 32076a0e9e..1eb6784f6a 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -216,7 +216,7 @@ sol1 = ahmc_bayesian_pinn_pde(pde_system, priorsNNw = (0.0, 1.0), saveats = [1 / 50.0], param = [LogNormal(6.0, 0.5)], - Dict_differentials = Dict_differentials) + Dict_differentials = Dict_differentials, progress = true) param = 2 * π ts = vec(sol1.timepoints[1]) @@ -227,69 +227,268 @@ u_predict = pmean(sol1.ensemblesol[1]) @test mean(u_predict .- u_real) < 0.1 @test sol1.estimated_de_params[1]≈param atol=param * 0.3 -println("Example 4: Lorenz System with New parameter estimation") -@parameters t, σ_ -@variables x(..), y(..), z(..) +println("Example 3: Lotka Volterra with New parameter estimation") +@parameters t α β γ δ +@variables x(..) y(..) + Dt = Differential(t) -eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), - Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), - Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] - -bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] -domains = [t ∈ Interval(0.0, 1.0)] - -input_ = length(domains) -n = 7 -chain = [ - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), +eqs = [Dt(x(t)) ~ α * x(t) - β * x(t) * y(t), Dt(y(t)) ~ -γ * y(t) + δ * x(t) * y(t)] +bcs = [x(0) ~ 1.0, y(0) ~ 1.0] +domains = [t ∈ Interval(0.0, 4.0)] + +# Define the parameters' values +# params = [α => 1.0, β => 0.5, γ => 0.5, δ => 1.0] +# p = [1.5, 1.0, 3.0, 1.0] + +chainl = [ + Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), + Lux.Dense(6, 1)), + Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), + Lux.Dense(6, 1)), ] -#Generate Data -function lorenz!(du, u, p, t) - du[1] = 10.0 * (u[2] - u[1]) - du[2] = u[1] * (28.0 - u[3]) - u[2] - du[3] = u[1] * u[2] - (8 / 3) * u[3] +initl, st = Lux.setup(Random.default_rng(), chainl[1]) +initl1, st1 = Lux.setup(Random.default_rng(), chainl[2]) + +@named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [x(t), y(t)], + [α, β, γ, δ], + defaults = Dict([α => 1.0, β => 0.5, γ => 0.5, δ => 1.0])) + +using NeuralPDE, Lux, Plots, OrdinaryDiffEq, Distributions, Random + +function lotka_volterra(u, p, t) + α, β, γ, δ = p + x, y = u + dx = (α - β * y) * x + dy = (δ * x - γ) * y + return [dx, dy] end -u0 = [1.0; 0.0; 0.0] -tspan = (0.0, 1.0) -prob = ODEProblem(lorenz!, u0, tspan) -sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) -ts = sol.t -us = hcat(sol.u...) -us = us .+ ((0.05 .* randn(size(us))) .* us) -ts_ = hcat(sol(ts).t...)[1, :] -dataset = [hcat(us[i, :], ts_) for i in 1:3] - -discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, +# initial-value problem. +u0 = [1.0, 1.0] +p = [1.5, 1.0, 3.0, 1.0] +tspan = (0.0, 4.0) +prob = ODEProblem(lotka_volterra, u0, tspan, p) + +# Solve using OrdinaryDiffEq.jl solver +dt = 0.05 +solution = solve(prob, Tsit5(); saveat = dt) + +# Extract solution +time = solution.t +u = hcat(solution.u...) +# plot(time, u[1, :]) +# plot!(time, u[2, :]) +# Construct dataset +dataset = [hcat(u[i, :], time) for i in 1:2] + +discretization = BayesianPINN(chainl, GridTraining(0.01), param_estim = true, dataset = [dataset, nothing]) -@named pde_system = PDESystem(eqs, bcs, domains, - [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) - # creating dictionary for masking equations eqs = pde_system.eqs Dict_differentials = Dict() exps = toexpr.(eqs) nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] +sol = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 500, + bcstd = [0.05, 0.05], + phystd = [0.005, 0.005], l2std = [0.1, 0.1], + priorsNNw = (0.0, 10.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2), + ], progress = true) + +# plot!(sol.timepoints[1]', sol.ensemblesol[1]) +# plot!(sol.timepoints[2]', sol.ensemblesol[2]) + sol1 = ahmc_bayesian_pinn_pde(pde_system, discretization; - draw_samples = 50, - bcstd = [0.3, 0.3, 0.3], - phystd = [0.1, 0.1, 0.1], - l2std = [1, 1, 1], - priorsNNw = (0.0, 1.0), - saveats = [0.01], - param = [Normal(12.0, 2)], - Dict_differentials = Dict_differentials) - -idealp = 10.0 -p_ = sol1.estimated_de_params[1] -@test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] -# @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] + draw_samples = 500, + bcstd = [0.05, 0.05], + phystd = [0.005, 0.005], l2std = [0.1, 0.1], + phystdnew = [0.1, 0.1], + # Kernel = AdvancedHMC.NUTS(0.8), + priorsNNw = (0.0, 10.0), + saveats = [1 / 50.0], + param = [ + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2), + ], + Dict_differentials = Dict_differentials, progress = true) + +# plot!(sol1.timepoints[1]', sol1.ensemblesol[1]) +# plot!(sol1.timepoints[2]', sol1.ensemblesol[2]) + +param = 2 * π +ts = vec(sol1.timepoints[1]) +u_real = [analytic_sol_func1(0.0, t) for t in ts] +u_predict = pmean(sol1.ensemblesol[1]) + +@test u_predict≈u_real atol=1.5 +@test mean(u_predict .- u_real) < 0.1 +@test sol1.estimated_de_params[1]≈param atol=param * 0.3 + +# points1 = [] +# for eq_arg in eq_args +# a = [] +# # for each (depvar,[indvar1..]) if indvari==indvar (eq_arg) +# for i in eachindex(symbols_input) +# if symbols_input[i][2] == eq_arg +# # include domain points of that depvar +# # each loss equation take domain matrix [points..;points..] +# push!(a, train_sets[i][:, 2:end]') +# end +# end +# # vcat as new row for next equation +# push!(points1, vcat(a...)) +# end +# println(points1 == points) + +using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC +import ModelingToolkit: Interval, infimum, supremum, Distributions +using Plots, MonteCarloMeasurements + +@parameters x, t, α +@variables u(..) +Dt = Differential(t) +Dx = Differential(x) +Dx2 = Differential(x)^2 +Dx3 = Differential(x)^3 +Dx4 = Differential(x)^4 + +# α = 1 +β = 4 +γ = 1 +eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 + +u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 + +bcs = [u(x, 0) ~ u_analytic(x, 0), + u(-10, t) ~ u_analytic(-10, t), + u(10, t) ~ u_analytic(10, t), + Dx(u(-10, t)) ~ du(-10, t), + Dx(u(10, t)) ~ du(10, t)] + +# Space and time domains +domains = [x ∈ Interval(-10.0, 10.0), + t ∈ Interval(0.0, 1.0)] + +# Discretization +dx = 0.4; +dt = 0.2; + +# Function to compute analytical solution at a specific point (x, t) +function u_analytic_point(x, t) + z = -x / 2 + t + return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +end + +# Function to generate the dataset matrix +function generate_dataset_matrix(domains, dx, dt) + x_values = -10:dx:10 + t_values = 0.0:dt:1.0 + + dataset = [] + + for t in t_values + for x in x_values + u_value = u_analytic_point(x, t) + push!(dataset, [u_value, x, t]) + end + end + + return vcat([data' for data in dataset]...) +end + +datasetpde = [generate_dataset_matrix(domains, dx, dt)] + +# noise to dataset +noisydataset = deepcopy(datasetpde) +noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ + randn(size(noisydataset[1][:, 1])) .* 5 / 100 .* + noisydataset[1][:, 1] + +# plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") +# plot!(noisydataset[1][:, 2], noisydataset[1][:, 1]) + +# Neural network +chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), + Lux.Dense(8, 8, Lux.tanh), + Lux.Dense(8, 1)) + +discretization = NeuralPDE.BayesianPINN([chain], + GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) + +@named pde_system = PDESystem(eq, + bcs, + domains, + [x, t], + [u(x, t)], + [α], + defaults = Dict([α => 0.5])) + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 100, + bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], + phystd = [1.0], l2std = [0.05], param = [Distributions.LogNormal(0.5, 2)], + priorsNNw = (0.0, 10.0), + saveats = [1 / 100.0, 1 / 100.0], progress = true) + +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 100, + bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], + phystd = [1.0], phystdnew = [0.05], l2std = [0.05], + param = [Distributions.LogNormal(0.5, 2)], + priorsNNw = (0.0, 10.0), + saveats = [1 / 100.0, 1 / 100.0], Dict_differentials = Dict_differentials, + progress = true) + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) for (d, dx) in zip(domains, [dx / 10, dt])] +u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +# p1 = plot(xs, u_predict, title = "predict") +# p2 = plot(xs, u_real, title = "analytic") +# p3 = plot(xs, diff_u, title = "error") +# plot(p1, p2, p3) + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) for (d, dx) in zip(domains, [dx / 10, dt])] +u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +# p1 = plot(xs, u_predict, title = "predict") +# p2 = plot(xs, u_real, title = "analytic") +# p3 = plot(xs, diff_u, title = "error") +# plot(p1, p2, p3) From 49dd7cbf4952d110e8d2ada4aa53b9f86f8ca2bb Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Tue, 27 Feb 2024 02:44:21 +0530 Subject: [PATCH 26/53] refactor code, Corrected PDE_BPINN Logphys calc. --- src/PDE_BPINN.jl | 26 ++- src/advancedHMC_MCMC.jl | 2 +- src/discretize.jl | 108 ++++++---- src/training_strategies.jl | 52 ++++- test/BPINN_PDEinvsol_tests.jl | 386 +++++++++++++++++++++------------- 5 files changed, 367 insertions(+), 207 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index 500510d64c..4375621782 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -3,8 +3,9 @@ mutable struct PDELogTargetDensity{ D <: Union{Nothing, Vector{<:Matrix{<:Real}}}, P <: Vector{<:Distribution}, I, - F, FF, - PH, + F, + FF, + PH } dim::Int64 strategy::ST @@ -26,8 +27,9 @@ mutable struct PDELogTargetDensity{ typeof(dataset), typeof(priors), typeof(init_params), - typeof(full_loglikelihood), typeof(L2_loss2), - typeof(Φ), + typeof(full_loglikelihood), + typeof(L2_loss2), + typeof(Φ) }(dim, strategy, dataset, @@ -36,7 +38,8 @@ mutable struct PDELogTargetDensity{ names, extraparams, init_params, - full_loglikelihood, L2_loss2, + full_loglikelihood, + L2_loss2, Φ) end function PDELogTargetDensity(dim, strategy, dataset, @@ -48,8 +51,9 @@ mutable struct PDELogTargetDensity{ typeof(dataset), typeof(priors), typeof(init_params), - typeof(full_loglikelihood), typeof(L2_loss2), - typeof(Φ), + typeof(full_loglikelihood), + typeof(L2_loss2), + typeof(Φ) }(dim, strategy, dataset, @@ -58,7 +62,8 @@ mutable struct PDELogTargetDensity{ names, extraparams, init_params, - full_loglikelihood, L2_loss2, + full_loglikelihood, + L2_loss2, Φ) end end @@ -374,7 +379,8 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; # consider all dataset domain points and for each row new set of equation loss function # this is a vector of tuple{vector,nothing} - pde_loss_functions = [merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, + pde_loss_functions = [merge_strategy_with_loglikelihood_function( + pinnrep::PINNRepresentation, GridTraining(0.1), yuh1[i], nothing; @@ -453,7 +459,7 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; # add init_params for NN params priors = [ MvNormal(priorsNNw[1] * ones(nparameters), - LinearAlgebra.Diagonal(abs2.(priorsNNw[2] .* ones(nparameters)))), + LinearAlgebra.Diagonal(abs2.(priorsNNw[2] .* ones(nparameters)))) ] # append Ode params to all paramvector - initial_θ diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index ea50eabcf6..9064ddd9fa 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -261,7 +261,7 @@ function innerdiff(Tar::LogTargetDensity, f, autodiff::Bool, t::AbstractVector, vals = nnsol .- physsol - # N dimensional vector if N outputs for NN(each row has logpdf of i[i] where u is vector of dependant variables) + # N dimensional vector if N outputs for NN(each row has logpdf of u[i] where u is vector of dependant variables) return [logpdf(MvNormal(vals[i, :], LinearAlgebra.Diagonal(abs2.(Tar.phystd[i] .* ones(length(vals[i, :]))))), diff --git a/src/discretize.jl b/src/discretize.jl index 0740544c09..dfb9b5174c 100644 --- a/src/discretize.jl +++ b/src/discretize.jl @@ -504,29 +504,30 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, bc_indvars, bc_integration_vars)] - pde_loss_functions, bc_loss_functions = merge_strategy_with_loss_function(pinnrep, - strategy, - datafree_pde_loss_functions, - datafree_bc_loss_functions) - # setup for all adaptive losses - num_pde_losses = length(pde_loss_functions) - num_bc_losses = length(bc_loss_functions) - # assume one single additional loss function if there is one. this means that the user needs to lump all their functions into a single one, - num_additional_loss = additional_loss isa Nothing ? 0 : 1 - - adaloss_T = eltype(adaloss.pde_loss_weights) - - # this will error if the user has provided a number of initial weights that is more than 1 and doesn't match the number of loss functions - adaloss.pde_loss_weights = ones(adaloss_T, num_pde_losses) .* adaloss.pde_loss_weights - adaloss.bc_loss_weights = ones(adaloss_T, num_bc_losses) .* adaloss.bc_loss_weights - adaloss.additional_loss_weights = ones(adaloss_T, num_additional_loss) .* - adaloss.additional_loss_weights - - reweight_losses_func = generate_adaptive_loss_function(pinnrep, adaloss, - pde_loss_functions, - bc_loss_functions) - function get_likelihood_estimate_function(discretization::PhysicsInformedNN) + pde_loss_functions, bc_loss_functions = merge_strategy_with_loss_function(pinnrep, + strategy, + datafree_pde_loss_functions, + datafree_bc_loss_functions) + # setup for all adaptive losses + num_pde_losses = length(pde_loss_functions) + num_bc_losses = length(bc_loss_functions) + # assume one single additional loss function if there is one. this means that the user needs to lump all their functions into a single one, + num_additional_loss = additional_loss isa Nothing ? 0 : 1 + + adaloss_T = eltype(adaloss.pde_loss_weights) + + # this will error if the user has provided a number of initial weights that is more than 1 and doesn't match the number of loss functions + adaloss.pde_loss_weights = ones(adaloss_T, num_pde_losses) .* + adaloss.pde_loss_weights + adaloss.bc_loss_weights = ones(adaloss_T, num_bc_losses) .* adaloss.bc_loss_weights + adaloss.additional_loss_weights = ones(adaloss_T, num_additional_loss) .* + adaloss.additional_loss_weights + + reweight_losses_func = generate_adaptive_loss_function(pinnrep, adaloss, + pde_loss_functions, + bc_loss_functions) + function full_loss_function(θ, p) # the aggregation happens on cpu even if the losses are gpu, probably fine since it's only a few of them pde_losses = [pde_loss_function(θ) for pde_loss_function in pde_loss_functions] @@ -603,46 +604,66 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, return full_weighted_loss end - return full_loss_function + return bc_loss_functions, pde_loss_functions, full_loss_function end function get_likelihood_estimate_function(discretization::BayesianPINN) + # Because seperate reweighting code section needed and loglikelihood is pointwise independant + pde_loss_functions, bc_loss_functions = merge_strategy_with_loglikelihood_function( + pinnrep, + strategy, + datafree_pde_loss_functions, + datafree_bc_loss_functions) + + # setup for all adaptive losses + num_pde_losses = length(pde_loss_functions) + num_bc_losses = length(bc_loss_functions) + # assume one single additional loss function if there is one. this means that the user needs to lump all their functions into a single one, + num_additional_loss = additional_loss isa Nothing ? 0 : 1 + + adaloss_T = eltype(adaloss.pde_loss_weights) + + # this will error if the user has provided a number of initial weights that is more than 1 and doesn't match the number of loss functions + adaloss.pde_loss_weights = ones(adaloss_T, num_pde_losses) .* + adaloss.pde_loss_weights + adaloss.bc_loss_weights = ones(adaloss_T, num_bc_losses) .* adaloss.bc_loss_weights + adaloss.additional_loss_weights = ones(adaloss_T, num_additional_loss) .* + adaloss.additional_loss_weights + + reweight_losses_func = generate_adaptive_loss_function(pinnrep, adaloss, + pde_loss_functions, + bc_loss_functions) + dataset_pde, dataset_bc = discretization.dataset dataset_pde = dataset_pde isa Nothing ? dataset_pde : get_dataset_train_points(eqs, dataset_pde, pinnrep) dataset_bc = dataset_bc isa Nothing ? dataset_bc : get_dataset_train_points(eqs, dataset_bc, pinnrep) # required as Physics loss also needed on the discrete dataset domain points # data points are discrete and so by default GridTraining loss applies - # passing placeholder dx with GridTraining, it uses data points irl - datapde_loss_functions, databc_loss_functions = if (!(dataset_bc isa Nothing)||!(dataset_pde isa Nothing)) - merge_strategy_with_loglikelihood_function(pinnrep, + # passing placeholder dx with GridTraining, it uses dataset points irl + datapde_loss_functions, databc_loss_functions = merge_strategy_with_loglikelihood_function( + pinnrep, GridTraining(0.1), datafree_pde_loss_functions, datafree_bc_loss_functions, train_sets_pde = dataset_pde, train_sets_bc = dataset_bc) - else - (nothing, nothing) - end function full_loss_function(θ, allstd::Vector{Vector{Float64}}) stdpdes, stdbcs, stdextra, stdpdesnew = allstd # the aggregation happens on cpu even if the losses are gpu, probably fine since it's only a few of them - pde_loglikelihoods = [logpdf(Normal(0, stdpdes[i]), pde_loss_function(θ)) - for (i, pde_loss_function) in enumerate(pde_loss_functions)] - - bc_loglikelihoods = [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) - for (j, bc_loss_function) in enumerate(bc_loss_functions)] - + # pde_loglikelihoods = sum([logpdf(MvNormal(pde_loss_function(θ)[1, :], LinearAlgebra.Diagonal(abs2.(stdpdes[i] .* ones(pde_loss_length[i])))), zeros(pde_loss_length[i])) for (i, pde_loss_function) in enumerate(pde_loss_functions)]) + pde_loglikelihoods = sum([pde_loss_function(θ, stdpdes[i]) for (i, pde_loss_function) in enumerate(pde_loss_functions)]) + # bc_loglikelihoods = sum([logpdf(MvNormal(bc_loss_function(θ)[1, :], LinearAlgebra.Diagonal(abs2.(stdbcs[j] .* ones(bc_loss_length[j])))), zeros(bc_loss_length[j])) for (j, bc_loss_function) in enumerate(bc_loss_functions)]) + bc_loglikelihoods = sum([bc_loss_function(θ, stdbcs[j]) for (j, bc_loss_function) in enumerate(bc_loss_functions)]) if !(datapde_loss_functions isa Nothing) - pde_loglikelihoods += [logpdf(Normal(0, stdpdes[j]), pde_loss_function(θ)) - for (j, pde_loss_function) in enumerate(datapde_loss_functions)] - + pde_loglikelihoods += sum([datapde_loss_function(θ, stdpdes[i]) for (i, datapde_loss_function) in enumerate(datapde_loss_functions)]) + # sum([logpdf(MvNormal(datapde_loss_function(θ)[1, :], LinearAlgebra.Diagonal(abs2.(stdpdes[i] .* ones(datapde_length[i])))), zeros(datapde_length[i])) for (i, datapde_loss_function) in enumerate(datapde_loss_functions)]) end if !(databc_loss_functions isa Nothing) - bc_loglikelihoods += [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) - for (j, bc_loss_function) in enumerate(databc_loss_functions)] + bc_loglikelihoods += sum([databc_loss_function(θ, stdbcs[j]) for (j, databc_loss_function) in enumerate(databc_loss_functions)]) + # sum([logpdf(MvNormal(databc_loss_function(θ)[1, :], LinearAlgebra.Diagonal(abs2.(stdbcs[j] .* ones(databc_length[j])))), zeros(databc_length[j])) for (j, databc_loss_function) in enumerate(databc_loss_functions)]) end # this is kind of a hack, and means that whenever the outer function is evaluated the increment goes up, even if it's not being optimized @@ -688,12 +709,13 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, return full_weighted_loglikelihood end - return full_loss_function + return bc_loss_functions, pde_loss_functions, full_loss_function end - full_loss_function = get_likelihood_estimate_function(discretization) + bc_loss_functions, pde_loss_functions, full_loss_function = get_likelihood_estimate_function(discretization) + pinnrep.loss_functions = PINNLossFunctions(bc_loss_functions, pde_loss_functions, - full_loss_function, additional_loss, + full_loss_function, additional_loss, datafree_pde_loss_functions, datafree_bc_loss_functions) diff --git a/src/training_strategies.jl b/src/training_strategies.jl index 1bf767beca..9fdff04f06 100644 --- a/src/training_strategies.jl +++ b/src/training_strategies.jl @@ -14,6 +14,7 @@ struct GridTraining{T} <: AbstractTrainingStrategy dx::T end +# dataset must have depvar values for same values of indvars function get_dataset_train_points(eqs, train_sets, pinnrep) dict_depvar_input = pinnrep.dict_depvar_input depvars = pinnrep.depvars @@ -23,11 +24,28 @@ function get_dataset_train_points(eqs, train_sets, pinnrep) symbols_input = [(i, dict_depvar_input[i]) for i in depvars] # [(:u, [:t])] eq_args = NeuralPDE.get_argument(eqs, dict_indvars, dict_depvars) - # [[:t]] - - points = [vcat([train_sets[i][:, 2:end]' - for i in eachindex(symbols_input) if symbols_input[i][2] == eq_arg]...) - for eq_arg in eq_args] + # equation wise indvar presence ~ [[:t]] + # in each equation atleast one depvars must be a function of all indvars(to cover heterogenous/not case) + + # train_sets follows order of depvars + # take dataset indvar values if for equations depvar's indvar matches input symbol indvar + # points = [vcat([train_sets[i][:, 2:end]' + # for i in eachindex(symbols_input) if symbols_input[i][2] == eq_arg]...) + # for eq_arg in eq_args] + + points = [] + for eq_arg in eq_args + eq_points = [] + for i in eachindex(symbols_input) + if symbols_input[i][2] == eq_arg + push!(eq_points, train_sets[i][:, 2:end]') + # Terminate to avoid repetitive ind var points inclusion + break + end + end + # Concatenate points for this equation argument + push!(points, vcat(eq_points...)) + end return points end @@ -36,17 +54,25 @@ end function merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, strategy::GridTraining, datafree_pde_loss_function, - datafree_bc_loss_function; train_sets_pde = nothing, train_sets_bc=nothing) + datafree_bc_loss_function; train_sets_pde = nothing, train_sets_bc = nothing) @unpack domains, eqs, bcs, dict_indvars, dict_depvars, flat_init_params = pinnrep - + dx = strategy.dx eltypeθ = eltype(pinnrep.flat_init_params) + # physics loss merge_strategy_with_loglikelihood_function call case + if ((train_sets_bc isa Nothing)&&(train_sets_pde isa Nothing)) + train_sets_pde, train_sets_bc = generate_training_sets( + domains, dx, eqs, bcs, eltypeθ, + dict_indvars, dict_depvars) + end + # is vec as later each _set in pde_train_sets are columns as points transformed to vector of points (pde_train_sets must be rowwise) pde_loss_functions = if !(train_sets_pde isa Nothing) + # dataset and domain pde losses case pde_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), train_sets_pde) - [get_loss_function(_loss, _set, eltypeθ, strategy) + [get_points_loss_functions(_loss, _set, eltypeθ, strategy) for (_loss, _set) in zip(datafree_pde_loss_function, pde_train_sets)] else @@ -54,10 +80,11 @@ function merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, end bc_loss_functions = if !(train_sets_bc isa Nothing) + # dataset and domain bc losses case bcs_train_sets = adapt.(parameterless_type(ComponentArrays.getdata(flat_init_params)), train_sets_bc) - [get_loss_function(_loss, _set, eltypeθ, strategy) + [get_points_loss_functions(_loss, _set, eltypeθ, strategy) for (_loss, _set) in zip(datafree_bc_loss_function, bcs_train_sets)] else nothing @@ -66,6 +93,13 @@ function merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, pde_loss_functions, bc_loss_functions end +function get_points_loss_functions(loss_function, train_set, eltypeθ, strategy::GridTraining; + τ = nothing) + function loss(θ, std) + logpdf(MvNormal(loss_function(train_set, θ)[1, :], LinearAlgebra.Diagonal(abs2.(std .* ones(length(train_set))))), zeros(length(train_set))) + end +end + function merge_strategy_with_loss_function(pinnrep::PINNRepresentation, strategy::GridTraining, datafree_pde_loss_function, diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 1eb6784f6a..250f98810b 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -112,7 +112,7 @@ end Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), Lux.Dense(n, 1)), Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), + Lux.Dense(n, 1)) ] #Generate Data @@ -216,7 +216,7 @@ sol1 = ahmc_bayesian_pinn_pde(pde_system, priorsNNw = (0.0, 1.0), saveats = [1 / 50.0], param = [LogNormal(6.0, 0.5)], - Dict_differentials = Dict_differentials, progress = true) + Dict_differentials = Dict_differentials) param = 2 * π ts = vec(sol1.timepoints[1]) @@ -234,7 +234,7 @@ println("Example 3: Lotka Volterra with New parameter estimation") Dt = Differential(t) eqs = [Dt(x(t)) ~ α * x(t) - β * x(t) * y(t), Dt(y(t)) ~ -γ * y(t) + δ * x(t) * y(t)] bcs = [x(0) ~ 1.0, y(0) ~ 1.0] -domains = [t ∈ Interval(0.0, 4.0)] +domains = [t ∈ Interval(0.0, 6.0)] # Define the parameters' values # params = [α => 1.0, β => 0.5, γ => 0.5, δ => 1.0] @@ -244,7 +244,7 @@ chainl = [ Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), Lux.Dense(6, 1)), Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), - Lux.Dense(6, 1)), + Lux.Dense(6, 1)) ] initl, st = Lux.setup(Random.default_rng(), chainl[1]) @@ -256,9 +256,9 @@ initl1, st1 = Lux.setup(Random.default_rng(), chainl[2]) [t], [x(t), y(t)], [α, β, γ, δ], - defaults = Dict([α => 1.0, β => 0.5, γ => 0.5, δ => 1.0])) + defaults = Dict([α => 5, β => 0, γ => 0.5, δ => 2])) -using NeuralPDE, Lux, Plots, OrdinaryDiffEq, Distributions, Random +using NeuralPDE, Lux, OrdinaryDiffEq, Distributions, Random function lotka_volterra(u, p, t) α, β, γ, δ = p @@ -281,10 +281,14 @@ solution = solve(prob, Tsit5(); saveat = dt) # Extract solution time = solution.t u = hcat(solution.u...) +u1 = u .+ ((0.3 .* randn(size(u))) .* u) + +# using Plots, StatsPlots +# plotly() # plot(time, u[1, :]) # plot!(time, u[2, :]) # Construct dataset -dataset = [hcat(u[i, :], time) for i in 1:2] +dataset = [hcat(u1[i, :], time) for i in 1:2] discretization = BayesianPINN(chainl, GridTraining(0.01), param_estim = true, dataset = [dataset, nothing]) @@ -295,6 +299,101 @@ Dict_differentials = Dict() exps = toexpr.(eqs) nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] +sol2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 500, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 3.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(0, 2), + Normal(0, 2), + Normal(0, 2), + Normal(0, 2) + ]) + +# plot(sol2.timepoints[1]', sol2.ensemblesol[1]) +# plot!(sol2.timepoints[2]', sol2.ensemblesol[2]) + +# plot(time, chainl[1](time', sol2.estimated_nn_params[1], st)[1]) +# plot!(time, chainl[2](time', sol2.estimated_nn_params[2], st)[1]) + +# sol1 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 500, +# bcstd = [0.1, 0.1], +# phystd = [0.1, 0.1], l2std = [0.1, 0.1], +# phystdnew = [1, 1], +# priorsNNw = (0.0, 3.0), +# saveats = [1 / 50.0], +# # Kernel = AdvancedHMC.NUTS(0.8), +# param = [ +# Normal(1, 2), +# Normal(2, 2), +# Normal(2, 2), +# Normal(0, 2) +# ], Dict_differentials = Dict_differentials, progress = true) + +# # plot(time', chainl[1](time', sol1.estimated_nn_params[1], st)[1]) +# # plot!(time, chainl[2](time', sol1.estimated_nn_params[2], st)[1]) + +# sol3 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 500, +# bcstd = [0.05, 0.05], +# phystd = [0.005, 0.005], l2std = [0.1, 0.1], +# phystdnew = [0.5, 0.5], +# # Kernel = AdvancedHMC.NUTS(0.8), +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 50.0], +# param = [ +# Normal(0.0, 2), +# Normal(0.0, 2), +# Normal(0.0, 2), +# Normal(0.0, 2) +# ], +# Dict_differentials = Dict_differentials, progress = true) + +# sol = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 500, +# bcstd = [0.05, 0.05], +# phystd = [0.005, 0.005], l2std = [0.1, 0.1], +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 50.0], +# # Kernel = AdvancedHMC.NUTS(0.8), +# param = [ +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2) +# ], progress = true) + +# plot!(sol.timepoints[1]', sol.ensemblesol[1]) +# plot!(sol.timepoints[2]', sol.ensemblesol[2]) + +# sol1 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 500, +# bcstd = [0.05, 0.05], +# phystd = [0.005, 0.005], l2std = [0.1, 0.1], +# phystdnew = [0.5, 0.5], +# # Kernel = AdvancedHMC.NUTS(0.8), +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 50.0], +# param = [ +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2) +# ], +# Dict_differentials = Dict_differentials, progress = true) + +# plot!(sol1.timepoints[1]', sol1.ensemblesol[1]) +# plot!(sol1.timepoints[2]', sol1.ensemblesol[2]) + sol = ahmc_bayesian_pinn_pde(pde_system, discretization; draw_samples = 500, @@ -307,8 +406,8 @@ sol = ahmc_bayesian_pinn_pde(pde_system, Normal(1.0, 2), Normal(1.0, 2), Normal(1.0, 2), - Normal(1.0, 2), - ], progress = true) + Normal(1.0, 2) + ]) # plot!(sol.timepoints[1]', sol.ensemblesol[1]) # plot!(sol.timepoints[2]', sol.ensemblesol[2]) @@ -318,7 +417,7 @@ sol1 = ahmc_bayesian_pinn_pde(pde_system, draw_samples = 500, bcstd = [0.05, 0.05], phystd = [0.005, 0.005], l2std = [0.1, 0.1], - phystdnew = [0.1, 0.1], + phystdnew = [0.5, 0.5], # Kernel = AdvancedHMC.NUTS(0.8), priorsNNw = (0.0, 10.0), saveats = [1 / 50.0], @@ -326,12 +425,9 @@ sol1 = ahmc_bayesian_pinn_pde(pde_system, Normal(1.0, 2), Normal(1.0, 2), Normal(1.0, 2), - Normal(1.0, 2), + Normal(1.0, 2) ], - Dict_differentials = Dict_differentials, progress = true) - -# plot!(sol1.timepoints[1]', sol1.ensemblesol[1]) -# plot!(sol1.timepoints[2]', sol1.ensemblesol[2]) + Dict_differentials = Dict_differentials) param = 2 * π ts = vec(sol1.timepoints[1]) @@ -358,137 +454,139 @@ u_predict = pmean(sol1.ensemblesol[1]) # end # println(points1 == points) -using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC -import ModelingToolkit: Interval, infimum, supremum, Distributions -using Plots, MonteCarloMeasurements - -@parameters x, t, α -@variables u(..) -Dt = Differential(t) -Dx = Differential(x) -Dx2 = Differential(x)^2 -Dx3 = Differential(x)^3 -Dx4 = Differential(x)^4 - -# α = 1 -β = 4 -γ = 1 -eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 - -u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 - -bcs = [u(x, 0) ~ u_analytic(x, 0), - u(-10, t) ~ u_analytic(-10, t), - u(10, t) ~ u_analytic(10, t), - Dx(u(-10, t)) ~ du(-10, t), - Dx(u(10, t)) ~ du(10, t)] - -# Space and time domains -domains = [x ∈ Interval(-10.0, 10.0), - t ∈ Interval(0.0, 1.0)] - -# Discretization -dx = 0.4; -dt = 0.2; - -# Function to compute analytical solution at a specific point (x, t) -function u_analytic_point(x, t) - z = -x / 2 + t - return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -end - -# Function to generate the dataset matrix -function generate_dataset_matrix(domains, dx, dt) - x_values = -10:dx:10 - t_values = 0.0:dt:1.0 - - dataset = [] - - for t in t_values - for x in x_values - u_value = u_analytic_point(x, t) - push!(dataset, [u_value, x, t]) - end - end - - return vcat([data' for data in dataset]...) -end - -datasetpde = [generate_dataset_matrix(domains, dx, dt)] - -# noise to dataset -noisydataset = deepcopy(datasetpde) -noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ - randn(size(noisydataset[1][:, 1])) .* 5 / 100 .* - noisydataset[1][:, 1] - -# plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") -# plot!(noisydataset[1][:, 2], noisydataset[1][:, 1]) - -# Neural network -chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), - Lux.Dense(8, 8, Lux.tanh), - Lux.Dense(8, 1)) +# using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC +# import ModelingToolkit: Interval, infimum, supremum, Distributions +# using Plots, MonteCarloMeasurements + +# @parameters x, t, α +# @variables u(..) +# Dt = Differential(t) +# Dx = Differential(x) +# Dx2 = Differential(x)^2 +# Dx3 = Differential(x)^3 +# Dx4 = Differential(x)^4 + +# # α = 1 +# β = 4 +# γ = 1 +# eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 + +# u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +# du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 + +# bcs = [u(x, 0) ~ u_analytic(x, 0), +# u(-10, t) ~ u_analytic(-10, t), +# u(10, t) ~ u_analytic(10, t), +# Dx(u(-10, t)) ~ du(-10, t), +# Dx(u(10, t)) ~ du(10, t)] + +# # Space and time domains +# domains = [x ∈ Interval(-10.0, 10.0), +# t ∈ Interval(0.0, 1.0)] + +# # Discretization +# dx = 0.4; +# dt = 0.2; + +# # Function to compute analytical solution at a specific point (x, t) +# function u_analytic_point(x, t) +# z = -x / 2 + t +# return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +# end -discretization = NeuralPDE.BayesianPINN([chain], - GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) +# # Function to generate the dataset matrix +# function generate_dataset_matrix(domains, dx, dt) +# x_values = -10:dx:10 +# t_values = 0.0:dt:1.0 -@named pde_system = PDESystem(eq, - bcs, - domains, - [x, t], - [u(x, t)], - [α], - defaults = Dict([α => 0.5])) +# dataset = [] -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 100, - bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], - phystd = [1.0], l2std = [0.05], param = [Distributions.LogNormal(0.5, 2)], - priorsNNw = (0.0, 10.0), - saveats = [1 / 100.0, 1 / 100.0], progress = true) +# for t in t_values +# for x in x_values +# u_value = u_analytic_point(x, t) +# push!(dataset, [u_value, x, t]) +# end +# end -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] +# return vcat([data' for data in dataset]...) +# end -sol2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 100, - bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], - phystd = [1.0], phystdnew = [0.05], l2std = [0.05], - param = [Distributions.LogNormal(0.5, 2)], - priorsNNw = (0.0, 10.0), - saveats = [1 / 100.0, 1 / 100.0], Dict_differentials = Dict_differentials, - progress = true) - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) for (d, dx) in zip(domains, [dx / 10, dt])] -u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -# p1 = plot(xs, u_predict, title = "predict") -# p2 = plot(xs, u_real, title = "analytic") -# p3 = plot(xs, diff_u, title = "error") -# plot(p1, p2, p3) - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) for (d, dx) in zip(domains, [dx / 10, dt])] -u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -# p1 = plot(xs, u_predict, title = "predict") -# p2 = plot(xs, u_real, title = "analytic") -# p3 = plot(xs, diff_u, title = "error") -# plot(p1, p2, p3) +# datasetpde = [generate_dataset_matrix(domains, dx, dt)] + +# # noise to dataset +# noisydataset = deepcopy(datasetpde) +# noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ +# randn(size(noisydataset[1][:, 1])) .* 5 / 100 .* +# noisydataset[1][:, 1] + +# # plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") +# # plot!(noisydataset[1][:, 2], noisydataset[1][:, 1]) + +# # Neural network +# chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), +# Lux.Dense(8, 8, Lux.tanh), +# Lux.Dense(8, 1)) + +# discretization = NeuralPDE.BayesianPINN([chain], +# GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) + +# @named pde_system = PDESystem(eq, +# bcs, +# domains, +# [x, t], +# [u(x, t)], +# [α], +# defaults = Dict([α => 0.5])) + +# sol1 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 100, +# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], +# phystd = [1.0], l2std = [0.05], param = [Distributions.LogNormal(0.5, 2)], +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 100.0, 1 / 100.0], progress = true) + +# eqs = pde_system.eqs +# Dict_differentials = Dict() +# exps = toexpr.(eqs) +# nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +# sol2 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 100, +# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], +# phystd = [1.0], phystdnew = [0.05], l2std = [0.05], +# param = [Distributions.LogNormal(0.5, 2)], +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 100.0, 1 / 100.0], Dict_differentials = Dict_differentials, +# progress = true) + +# phi = discretization.phi[1] +# xs, ts = [infimum(d.domain):dx:supremum(d.domain) +# for (d, dx) in zip(domains, [dx / 10, dt])] +# u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] +# for t in ts] +# u_real = [[u_analytic(x, t) for x in xs] for t in ts] +# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) +# for x in xs] +# for t in ts] + +# # p1 = plot(xs, u_predict, title = "predict") +# # p2 = plot(xs, u_real, title = "analytic") +# # p3 = plot(xs, diff_u, title = "error") +# # plot(p1, p2, p3) + +# phi = discretization.phi[1] +# xs, ts = [infimum(d.domain):dx:supremum(d.domain) +# for (d, dx) in zip(domains, [dx / 10, dt])] +# u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] +# for t in ts] +# u_real = [[u_analytic(x, t) for x in xs] for t in ts] +# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) +# for x in xs] +# for t in ts] + +# # p1 = plot(xs, u_predict, title = "predict") +# # p2 = plot(xs, u_real, title = "analytic") +# # p3 = plot(xs, diff_u, title = "error") +# # plot(p1, p2, p3) From 014a11d6cb2a7fdcd688847cb3dc9ad03f975118 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 29 Feb 2024 00:12:00 +0530 Subject: [PATCH 27/53] corrected original and new implementation, comments --- src/PDE_BPINN.jl | 96 ++++---- src/discretize.jl | 21 +- src/training_strategies.jl | 11 +- test/BPINN_PDE_tests.jl | 21 +- test/BPINN_PDEinvsol_tests.jl | 447 +--------------------------------- 5 files changed, 82 insertions(+), 514 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index 4375621782..b18c35aa99 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -68,63 +68,66 @@ mutable struct PDELogTargetDensity{ end end -# for bc case, [bc]/bc eqs must be passed along with dataset_bc[i] -# and final loss for bc must be together in a vector(bcs has separate type of dataset_bc) -# eqs is vector of pde eqs and dataset here is dataset_pde -# normally you get vector of losses +# you get a vector of losses function get_lossy(pinnrep, dataset, Dict_differentials) eqs = pinnrep.eqs - depvars = pinnrep.depvars # order is same as dataset and interps + depvars = pinnrep.depvars #depvar order is same as dataset # Dict_differentials is filled with Differential operator => diff_i key-value pairs # masking operation eqs_new = substitute.(eqs, Ref(Dict_differentials)) to_subs, tobe_subs = get_symbols(dataset, depvars, eqs) - # for each row in dataset create u values for substituting in equation, n_equations=n_rows + + # for values of all depvars at corresponding indvar values in dataset, create dictionaries {Dict(x(t) => 1.0496435863173237, y(t) => 1.9227770685615337)} + # In each Dict, num form of depvar is key to its value at certain coords of indvars, n_dicts = n_rows_dataset(or n_indvar_coords_dataset) eq_subs = [Dict(tobe_subs[depvar] => to_subs[depvar][i] for depvar in depvars) for i in 1:size(dataset[1][:, 1])[1]] - # for each point(eq_sub dictionary) substiute in all equations(eqs_new - masked equations) - b = [[substitute(eq, eq_sub) for eq in eqs_new] for eq_sub in eq_subs] - - # now we have vector of equation vectors + # for each dataset point(eq_sub dictionary), substitute in masked equations + # n_collocated_equations = n_rows_dataset(or n_indvar_coords_dataset) + masked_colloc_equations = [[substitute(eq, eq_sub) for eq in eqs_new] + for eq_sub in eq_subs] + # now we have vector of dataset depvar's collocated equations # reverse dict for re-substituting values of Differential(t)(u(t)) etc rev_Dict_differentials = Dict(value => key for (key, value) in Dict_differentials) - # for each vector in vector of equation vectorbroadcast resubstituing OG mask values - c = [substitute.(b_i, Ref(rev_Dict_differentials)) for b_i in b] + # unmask Differential terms in masked_colloc_equations + colloc_equations = [substitute.(masked_colloc_equation, Ref(rev_Dict_differentials)) + for masked_colloc_equation in masked_colloc_equations] - # get losses, zip each equation with args for each build_loss call per equation vector - loss_functions = [[build_loss_function(pinnrep, eq, pde_indvar) - for (eq, pde_indvar, integration_indvar) in zip(c[i], + # nested vector of datafree_pde_loss_functions (as in discretize.jl) + # each sub vector has dataset's indvar coord's datafree_colloc_loss_function, n_subvectors = n_rows_dataset(or n_indvar_coords_dataset) + # zip each colloc equation with args for each build_loss call per equation vector + datafree_colloc_loss_functions = [[build_loss_function(pinnrep, eq, pde_indvar) + for (eq, pde_indvar, integration_indvar) in zip(colloc_equation, pinnrep.pde_indvars, - pinnrep.pde_integration_vars)] for i in eachindex(c)] + pinnrep.pde_integration_vars)] for colloc_equation in colloc_equations] - return loss_functions + return datafree_colloc_loss_functions end -# dataset_pde has normal matrix format -# dataset_bc has format of Vector{typeof(dataset_pde )} as each bc has different domain requirements function get_symbols(dataset, depvars, eqs) + # take only values of depvars from dataset depvar_vals = [dataset_i[:, 1] for dataset_i in dataset] - # order of depvars + # order of pinnrep.depvars, depvar_vals, BayesianPINN.dataset must be same to_subs = Dict(depvars .=> depvar_vals) - asrt = Symbolics.get_variables.(eqs) - # want only symbols of depvars - temp = unique(reduce(vcat, asrt)) + numform_vars = Symbolics.get_variables.(eqs) + Eq_vars = unique(reduce(vcat, numform_vars)) + # got equation's depvar num format {x(t)} for use in substitute() tobe_subs = Dict() for a in depvars - for i in temp + for i in Eq_vars expr = toexpr(i) if (expr isa Expr) && (expr.args[1] == a) tobe_subs[a] = i end end end + # depvar symbolic and num format got, tobe_subs : Dict{Any, Any}(:y => y(t), :x => x(t)) return to_subs, tobe_subs end @@ -373,37 +376,39 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; newloss = if Dict_differentials isa Nothing nothing else - yuh1 = get_lossy(pinnrep, dataset_pde, Dict_differentials) - # eqs = pinnrep.bcs - # yuh2 = get_lossy(pinnrep, dataset_pde, eqs) - - # consider all dataset domain points and for each row new set of equation loss function - # this is a vector of tuple{vector,nothing} - pde_loss_functions = [merge_strategy_with_loglikelihood_function( - pinnrep::PINNRepresentation, + datafree_colloc_loss_functions = get_lossy(pinnrep, dataset_pde, Dict_differentials) + # equals number of indvar coords in dataset + # add case for if parameters present in bcs? + + train_sets_pde = get_dataset_train_points(pde_system.eqs, + dataset_pde, + pinnrep) + colloc_train_sets = [[hcat(train_sets_pde[i][:, j]...)' for i in eachindex(datafree_colloc_loss_functions[1])] for j in eachindex(datafree_colloc_loss_functions)] + + # for each datafree_colloc_loss_function create loss_functions by passing dataset's indvar coords as train_sets_pde. + # placeholder strategy = GridTraining(0.1), datafree_bc_loss_function and train_sets_bc must be nothing + # order of indvar coords will be same as corresponding depvar coords values in dataset provided in get_lossy() call. + pde_loss_function_points = [merge_strategy_with_loglikelihood_function( + pinnrep, GridTraining(0.1), - yuh1[i], + datafree_colloc_loss_functions[i], nothing; - # pass transformation of each dataset row-corresponds to each point, for each depvar dataset point merged equation vector - train_sets_pde = get_dataset_train_points(pde_system.eqs, - [Array(data[i, :]') for data in dataset_pde], - pinnrep), + train_sets_pde = colloc_train_sets[i], train_sets_bc = nothing) - for i in eachindex(yuh1)] + for i in eachindex(datafree_colloc_loss_functions)] function L2_loss2(θ, allstd) stdpdesnew = allstd[4] # first vector of losses,from tuple -> pde losses, first[1] pde loss - pde_loglikelihoods = [[logpdf(Normal(0, stdpdesnew[j]), pde_loss_function(θ)) - for (j, pde_loss_function) in enumerate(pde_loss_functions[i][1])] - for i in eachindex(pde_loss_functions)] + pde_loglikelihoods = [sum([pde_loss_function(θ, stdpdesnew[i]) + for (i, pde_loss_function) in enumerate(pde_loss_functions[1])]) + for pde_loss_functions in pde_loss_function_points] - # bc_loglikelihoods = [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) + # bc_loglikelihoods = [sum([bc_loss_function(θ, stdpdesnew[i]) for (i, bc_loss_function) in enumerate(pde_loss_function_points[1])]) for pde_loss_function_points in pde_loss_functions] # for (j, bc_loss_function) in enumerate(bc_loss_functions)] - return sum(sum(pde_loglikelihoods)) - # sum(sum(pde_loglikelihoods) + sum(bc_loglikelihoods)) + return sum(pde_loglikelihoods) end end @@ -437,9 +442,6 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; # NN solutions for loglikelihood which is used for L2lossdata Φ = pinnrep.phi - # for new L2 loss - # discretization.additional_loss = - if nchains < 1 throw(error("number of chains must be greater than or equal to 1")) end diff --git a/src/discretize.jl b/src/discretize.jl index dfb9b5174c..76378adbeb 100644 --- a/src/discretize.jl +++ b/src/discretize.jl @@ -608,7 +608,7 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, end function get_likelihood_estimate_function(discretization::BayesianPINN) - # Because seperate reweighting code section needed and loglikelihood is pointwise independant + # Because separate reweighting code section needed and loglikelihood is pointwise independent pde_loss_functions, bc_loss_functions = merge_strategy_with_loglikelihood_function( pinnrep, strategy, @@ -652,18 +652,19 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, function full_loss_function(θ, allstd::Vector{Vector{Float64}}) stdpdes, stdbcs, stdextra, stdpdesnew = allstd # the aggregation happens on cpu even if the losses are gpu, probably fine since it's only a few of them - # pde_loglikelihoods = sum([logpdf(MvNormal(pde_loss_function(θ)[1, :], LinearAlgebra.Diagonal(abs2.(stdpdes[i] .* ones(pde_loss_length[i])))), zeros(pde_loss_length[i])) for (i, pde_loss_function) in enumerate(pde_loss_functions)]) - pde_loglikelihoods = sum([pde_loss_function(θ, stdpdes[i]) for (i, pde_loss_function) in enumerate(pde_loss_functions)]) - # bc_loglikelihoods = sum([logpdf(MvNormal(bc_loss_function(θ)[1, :], LinearAlgebra.Diagonal(abs2.(stdbcs[j] .* ones(bc_loss_length[j])))), zeros(bc_loss_length[j])) for (j, bc_loss_function) in enumerate(bc_loss_functions)]) - bc_loglikelihoods = sum([bc_loss_function(θ, stdbcs[j]) for (j, bc_loss_function) in enumerate(bc_loss_functions)]) + + pde_loglikelihoods = sum([pde_loss_function(θ, stdpdes[i]) + for (i, pde_loss_function) in enumerate(pde_loss_functions)]) + bc_loglikelihoods = sum([bc_loss_function(θ, stdbcs[j]) + for (j, bc_loss_function) in enumerate(bc_loss_functions)]) + if !(datapde_loss_functions isa Nothing) - pde_loglikelihoods += sum([datapde_loss_function(θ, stdpdes[i]) for (i, datapde_loss_function) in enumerate(datapde_loss_functions)]) - # sum([logpdf(MvNormal(datapde_loss_function(θ)[1, :], LinearAlgebra.Diagonal(abs2.(stdpdes[i] .* ones(datapde_length[i])))), zeros(datapde_length[i])) for (i, datapde_loss_function) in enumerate(datapde_loss_functions)]) + pde_loglikelihoods += sum([datapde_loss_function(θ, stdpdes[i]) + for (i, datapde_loss_function) in enumerate(datapde_loss_functions)]) end - if !(databc_loss_functions isa Nothing) - bc_loglikelihoods += sum([databc_loss_function(θ, stdbcs[j]) for (j, databc_loss_function) in enumerate(databc_loss_functions)]) - # sum([logpdf(MvNormal(databc_loss_function(θ)[1, :], LinearAlgebra.Diagonal(abs2.(stdbcs[j] .* ones(databc_length[j])))), zeros(databc_length[j])) for (j, databc_loss_function) in enumerate(databc_loss_functions)]) + bc_loglikelihoods += sum([databc_loss_function(θ, stdbcs[j]) + for (j, databc_loss_function) in enumerate(databc_loss_functions)]) end # this is kind of a hack, and means that whenever the outer function is evaluated the increment goes up, even if it's not being optimized diff --git a/src/training_strategies.jl b/src/training_strategies.jl index 9fdff04f06..660b7cb2fb 100644 --- a/src/training_strategies.jl +++ b/src/training_strategies.jl @@ -29,10 +29,6 @@ function get_dataset_train_points(eqs, train_sets, pinnrep) # train_sets follows order of depvars # take dataset indvar values if for equations depvar's indvar matches input symbol indvar - # points = [vcat([train_sets[i][:, 2:end]' - # for i in eachindex(symbols_input) if symbols_input[i][2] == eq_arg]...) - # for eq_arg in eq_args] - points = [] for eq_arg in eq_args eq_points = [] @@ -95,8 +91,13 @@ end function get_points_loss_functions(loss_function, train_set, eltypeθ, strategy::GridTraining; τ = nothing) + # loss_function length is number of all points loss is being evaluated upon + # train sets rows are for each indvar, cols are coordinates (row_1,row_2,..row_n) at which loss evaluated function loss(θ, std) - logpdf(MvNormal(loss_function(train_set, θ)[1, :], LinearAlgebra.Diagonal(abs2.(std .* ones(length(train_set))))), zeros(length(train_set))) + logpdf( + MvNormal(loss_function(train_set, θ)[1, :], + LinearAlgebra.Diagonal(abs2.(std .* ones(size(train_set)[2])))), + zeros(size(train_set)[2])) end end diff --git a/test/BPINN_PDE_tests.jl b/test/BPINN_PDE_tests.jl index 6dd3637f5a..7923a14daf 100644 --- a/test/BPINN_PDE_tests.jl +++ b/test/BPINN_PDE_tests.jl @@ -8,7 +8,7 @@ using Flux Random.seed!(100) -@testset "Example 1: 2D Periodic System" begin +@testset "Example 1: 1D Periodic System" begin # Cos(pi*t) example @parameters t @variables u(..) @@ -35,8 +35,9 @@ Random.seed!(100) ts = vec(sol1.timepoints[1]) u_real = [analytic_sol_func(0.0, t) for t in ts] u_predict = pmean(sol1.ensemblesol[1]) - @test u_predict≈u_real atol=0.5 - @test mean(u_predict .- u_real) < 0.1 + + @test u_predict≈u_real atol=0.05 + @test mean(u_predict .- u_real) < 0.001 end @testset "Example 2: 1D ODE" begin @@ -67,13 +68,13 @@ end bcstd = [0.1], phystd = [0.05], priorsNNw = (0.0, 10.0), - saveats = [1 / 100.0]) + saveats = [1 / 100.0], progress=true) analytic_sol_func(t) = exp(-(t^2) / 2) / (1 + t + t^3) + t^2 ts = sol1.timepoints[1] u_real = vec([analytic_sol_func(t) for t in ts]) u_predict = pmean(sol1.ensemblesol[1]) - @test u_predict≈u_real atol=0.8 + @test u_predict≈u_real atol=0.5 end @testset "Example 3: 3rd Degree ODE" begin @@ -156,9 +157,9 @@ end sol1 = ahmc_bayesian_pinn_pde(pde_system, discretization; draw_samples = 200, - bcstd = [0.003, 0.003, 0.003, 0.003], - phystd = [0.003], - priorsNNw = (0.0, 10.0), + bcstd = [0.01, 0.01, 0.01, 0.01], + phystd = [0.005], + priorsNNw = (0.0, 2.0), saveats = [1 / 100.0, 1 / 100.0]) xs = sol1.timepoints[1] @@ -166,7 +167,7 @@ end u_predict = pmean(sol1.ensemblesol[1]) u_real = [analytic_sol_func(xs[:, i][1], xs[:, i][2]) for i in 1:length(xs[1, :])] - @test u_predict≈u_real atol=1.5 + @test u_predict≈u_real atol=0.8 end @testset "Translating from Flux" begin @@ -204,5 +205,5 @@ end ts = sol1.timepoints[1] u_real = vec([analytic_sol_func(t) for t in ts]) u_predict = pmean(sol1.ensemblesol[1]) - @test u_predict≈u_real atol=0.8 + @test u_predict≈u_real atol=0.1 end diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 250f98810b..837a5cc79b 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -7,7 +7,7 @@ using ComponentArrays, ModelingToolkit Random.seed!(100) -@testset "Example 1: 2D Periodic System with parameter estimation" begin +@testset "Example 1: 1D Periodic System with parameter estimation" begin # Cos(pi*t) periodic curve @parameters t, p @variables u(..) @@ -88,9 +88,9 @@ Random.seed!(100) u_real = [analytic_sol_func1(0.0, t) for t in ts] u_predict = pmean(sol1.ensemblesol[1]) - @test u_predict≈u_real atol=1.5 - @test mean(u_predict .- u_real) < 0.1 - @test sol1.estimated_de_params[1]≈param atol=param * 0.3 + @test u_predict≈u_real atol=0.1 + @test mean(u_predict .- u_real) < 0.01 + @test sol1.estimated_de_params[1]≈param atol=0.1 end @testset "Example 2: Lorenz System with parameter estimation" begin @@ -152,441 +152,4 @@ end p_ = sol1.estimated_de_params[1] @test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] # @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] -end - -function recur_expression(exp, Dict_differentials) - for in_exp in exp.args - if !(in_exp isa Expr) - # skip +,== symbols, characters etc - continue - - elseif in_exp.args[1] isa ModelingToolkit.Differential - # first symbol of differential term - # Dict_differentials for masking differential terms - # and resubstituting differentials in equations after putting in interpolations - # temp = in_exp.args[end] - Dict_differentials[eval(in_exp)] = Symbolics.variable("diff_$(length(Dict_differentials) + 1)") - return - else - recur_expression(in_exp, Dict_differentials) - end - end -end - -println("Example 3: 2D Periodic System with New parameter estimation") -@parameters t, p -@variables u(..) - -Dt = Differential(t) -eqs = Dt(u(t)) - cos(p * t) ~ 0 -bcs = [u(0) ~ 0.0] -domains = [t ∈ Interval(0.0, 2.0)] - -chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) -initl, st = Lux.setup(Random.default_rng(), chainl) - -@named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [u(t)], - [p], - defaults = Dict([p => 4.0])) - -analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) -timepoints = collect(0.0:(1 / 100.0):2.0) -u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] -u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) -dataset = [hcat(u1, timepoints)] - -discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, - dataset = [dataset, nothing]) - -# creating dictionary for masking equations -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)], - Dict_differentials = Dict_differentials) - -param = 2 * π -ts = vec(sol1.timepoints[1]) -u_real = [analytic_sol_func1(0.0, t) for t in ts] -u_predict = pmean(sol1.ensemblesol[1]) - -@test u_predict≈u_real atol=1.5 -@test mean(u_predict .- u_real) < 0.1 -@test sol1.estimated_de_params[1]≈param atol=param * 0.3 - -println("Example 3: Lotka Volterra with New parameter estimation") -@parameters t α β γ δ -@variables x(..) y(..) - -Dt = Differential(t) -eqs = [Dt(x(t)) ~ α * x(t) - β * x(t) * y(t), Dt(y(t)) ~ -γ * y(t) + δ * x(t) * y(t)] -bcs = [x(0) ~ 1.0, y(0) ~ 1.0] -domains = [t ∈ Interval(0.0, 6.0)] - -# Define the parameters' values -# params = [α => 1.0, β => 0.5, γ => 0.5, δ => 1.0] -# p = [1.5, 1.0, 3.0, 1.0] - -chainl = [ - Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), - Lux.Dense(6, 1)), - Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), - Lux.Dense(6, 1)) -] - -initl, st = Lux.setup(Random.default_rng(), chainl[1]) -initl1, st1 = Lux.setup(Random.default_rng(), chainl[2]) - -@named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [x(t), y(t)], - [α, β, γ, δ], - defaults = Dict([α => 5, β => 0, γ => 0.5, δ => 2])) - -using NeuralPDE, Lux, OrdinaryDiffEq, Distributions, Random - -function lotka_volterra(u, p, t) - α, β, γ, δ = p - x, y = u - dx = (α - β * y) * x - dy = (δ * x - γ) * y - return [dx, dy] -end - -# initial-value problem. -u0 = [1.0, 1.0] -p = [1.5, 1.0, 3.0, 1.0] -tspan = (0.0, 4.0) -prob = ODEProblem(lotka_volterra, u0, tspan, p) - -# Solve using OrdinaryDiffEq.jl solver -dt = 0.05 -solution = solve(prob, Tsit5(); saveat = dt) - -# Extract solution -time = solution.t -u = hcat(solution.u...) -u1 = u .+ ((0.3 .* randn(size(u))) .* u) - -# using Plots, StatsPlots -# plotly() -# plot(time, u[1, :]) -# plot!(time, u[2, :]) -# Construct dataset -dataset = [hcat(u1[i, :], time) for i in 1:2] - -discretization = BayesianPINN(chainl, GridTraining(0.01), param_estim = true, - dataset = [dataset, nothing]) - -# creating dictionary for masking equations -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 500, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 3.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(0, 2), - Normal(0, 2), - Normal(0, 2), - Normal(0, 2) - ]) - -# plot(sol2.timepoints[1]', sol2.ensemblesol[1]) -# plot!(sol2.timepoints[2]', sol2.ensemblesol[2]) - -# plot(time, chainl[1](time', sol2.estimated_nn_params[1], st)[1]) -# plot!(time, chainl[2](time', sol2.estimated_nn_params[2], st)[1]) - -# sol1 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 500, -# bcstd = [0.1, 0.1], -# phystd = [0.1, 0.1], l2std = [0.1, 0.1], -# phystdnew = [1, 1], -# priorsNNw = (0.0, 3.0), -# saveats = [1 / 50.0], -# # Kernel = AdvancedHMC.NUTS(0.8), -# param = [ -# Normal(1, 2), -# Normal(2, 2), -# Normal(2, 2), -# Normal(0, 2) -# ], Dict_differentials = Dict_differentials, progress = true) - -# # plot(time', chainl[1](time', sol1.estimated_nn_params[1], st)[1]) -# # plot!(time, chainl[2](time', sol1.estimated_nn_params[2], st)[1]) - -# sol3 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 500, -# bcstd = [0.05, 0.05], -# phystd = [0.005, 0.005], l2std = [0.1, 0.1], -# phystdnew = [0.5, 0.5], -# # Kernel = AdvancedHMC.NUTS(0.8), -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 50.0], -# param = [ -# Normal(0.0, 2), -# Normal(0.0, 2), -# Normal(0.0, 2), -# Normal(0.0, 2) -# ], -# Dict_differentials = Dict_differentials, progress = true) - -# sol = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 500, -# bcstd = [0.05, 0.05], -# phystd = [0.005, 0.005], l2std = [0.1, 0.1], -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 50.0], -# # Kernel = AdvancedHMC.NUTS(0.8), -# param = [ -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2) -# ], progress = true) - -# plot!(sol.timepoints[1]', sol.ensemblesol[1]) -# plot!(sol.timepoints[2]', sol.ensemblesol[2]) - -# sol1 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 500, -# bcstd = [0.05, 0.05], -# phystd = [0.005, 0.005], l2std = [0.1, 0.1], -# phystdnew = [0.5, 0.5], -# # Kernel = AdvancedHMC.NUTS(0.8), -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 50.0], -# param = [ -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2) -# ], -# Dict_differentials = Dict_differentials, progress = true) - -# plot!(sol1.timepoints[1]', sol1.ensemblesol[1]) -# plot!(sol1.timepoints[2]', sol1.ensemblesol[2]) - -sol = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 500, - bcstd = [0.05, 0.05], - phystd = [0.005, 0.005], l2std = [0.1, 0.1], - priorsNNw = (0.0, 10.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2) - ]) - -# plot!(sol.timepoints[1]', sol.ensemblesol[1]) -# plot!(sol.timepoints[2]', sol.ensemblesol[2]) - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 500, - bcstd = [0.05, 0.05], - phystd = [0.005, 0.005], l2std = [0.1, 0.1], - phystdnew = [0.5, 0.5], - # Kernel = AdvancedHMC.NUTS(0.8), - priorsNNw = (0.0, 10.0), - saveats = [1 / 50.0], - param = [ - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2) - ], - Dict_differentials = Dict_differentials) - -param = 2 * π -ts = vec(sol1.timepoints[1]) -u_real = [analytic_sol_func1(0.0, t) for t in ts] -u_predict = pmean(sol1.ensemblesol[1]) - -@test u_predict≈u_real atol=1.5 -@test mean(u_predict .- u_real) < 0.1 -@test sol1.estimated_de_params[1]≈param atol=param * 0.3 - -# points1 = [] -# for eq_arg in eq_args -# a = [] -# # for each (depvar,[indvar1..]) if indvari==indvar (eq_arg) -# for i in eachindex(symbols_input) -# if symbols_input[i][2] == eq_arg -# # include domain points of that depvar -# # each loss equation take domain matrix [points..;points..] -# push!(a, train_sets[i][:, 2:end]') -# end -# end -# # vcat as new row for next equation -# push!(points1, vcat(a...)) -# end -# println(points1 == points) - -# using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC -# import ModelingToolkit: Interval, infimum, supremum, Distributions -# using Plots, MonteCarloMeasurements - -# @parameters x, t, α -# @variables u(..) -# Dt = Differential(t) -# Dx = Differential(x) -# Dx2 = Differential(x)^2 -# Dx3 = Differential(x)^3 -# Dx4 = Differential(x)^4 - -# # α = 1 -# β = 4 -# γ = 1 -# eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 - -# u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -# du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 - -# bcs = [u(x, 0) ~ u_analytic(x, 0), -# u(-10, t) ~ u_analytic(-10, t), -# u(10, t) ~ u_analytic(10, t), -# Dx(u(-10, t)) ~ du(-10, t), -# Dx(u(10, t)) ~ du(10, t)] - -# # Space and time domains -# domains = [x ∈ Interval(-10.0, 10.0), -# t ∈ Interval(0.0, 1.0)] - -# # Discretization -# dx = 0.4; -# dt = 0.2; - -# # Function to compute analytical solution at a specific point (x, t) -# function u_analytic_point(x, t) -# z = -x / 2 + t -# return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -# end - -# # Function to generate the dataset matrix -# function generate_dataset_matrix(domains, dx, dt) -# x_values = -10:dx:10 -# t_values = 0.0:dt:1.0 - -# dataset = [] - -# for t in t_values -# for x in x_values -# u_value = u_analytic_point(x, t) -# push!(dataset, [u_value, x, t]) -# end -# end - -# return vcat([data' for data in dataset]...) -# end - -# datasetpde = [generate_dataset_matrix(domains, dx, dt)] - -# # noise to dataset -# noisydataset = deepcopy(datasetpde) -# noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ -# randn(size(noisydataset[1][:, 1])) .* 5 / 100 .* -# noisydataset[1][:, 1] - -# # plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") -# # plot!(noisydataset[1][:, 2], noisydataset[1][:, 1]) - -# # Neural network -# chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), -# Lux.Dense(8, 8, Lux.tanh), -# Lux.Dense(8, 1)) - -# discretization = NeuralPDE.BayesianPINN([chain], -# GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) - -# @named pde_system = PDESystem(eq, -# bcs, -# domains, -# [x, t], -# [u(x, t)], -# [α], -# defaults = Dict([α => 0.5])) - -# sol1 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 100, -# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], -# phystd = [1.0], l2std = [0.05], param = [Distributions.LogNormal(0.5, 2)], -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 100.0, 1 / 100.0], progress = true) - -# eqs = pde_system.eqs -# Dict_differentials = Dict() -# exps = toexpr.(eqs) -# nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -# sol2 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 100, -# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], -# phystd = [1.0], phystdnew = [0.05], l2std = [0.05], -# param = [Distributions.LogNormal(0.5, 2)], -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 100.0, 1 / 100.0], Dict_differentials = Dict_differentials, -# progress = true) - -# phi = discretization.phi[1] -# xs, ts = [infimum(d.domain):dx:supremum(d.domain) -# for (d, dx) in zip(domains, [dx / 10, dt])] -# u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] -# for t in ts] -# u_real = [[u_analytic(x, t) for x in xs] for t in ts] -# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) -# for x in xs] -# for t in ts] - -# # p1 = plot(xs, u_predict, title = "predict") -# # p2 = plot(xs, u_real, title = "analytic") -# # p3 = plot(xs, diff_u, title = "error") -# # plot(p1, p2, p3) - -# phi = discretization.phi[1] -# xs, ts = [infimum(d.domain):dx:supremum(d.domain) -# for (d, dx) in zip(domains, [dx / 10, dt])] -# u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] -# for t in ts] -# u_real = [[u_analytic(x, t) for x in xs] for t in ts] -# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) -# for x in xs] -# for t in ts] - -# # p1 = plot(xs, u_predict, title = "predict") -# # p2 = plot(xs, u_real, title = "analytic") -# # p3 = plot(xs, diff_u, title = "error") -# # plot(p1, p2, p3) +end \ No newline at end of file From 11bbba7d4cd11fe0b0c5cf0886307955e603f550 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 29 Feb 2024 00:44:59 +0530 Subject: [PATCH 28/53] update BPINN_ode, BPINN_PDE_tests --- src/BPINN_ode.jl | 3 --- test/BPINN_PDE_tests.jl | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/BPINN_ode.jl b/src/BPINN_ode.jl index 7e39c24eef..f092d043c5 100644 --- a/src/BPINN_ode.jl +++ b/src/BPINN_ode.jl @@ -217,9 +217,6 @@ function DiffEqBase.__solve(prob::DiffEqBase.ODEProblem, if chain isa Lux.AbstractExplicitLayer θinit, st = Lux.setup(Random.default_rng(), chain) - println(length(θinit)) - println(length(samples[1])) - println(draw_samples) θ = [vector_to_parameters(samples[i][1:(end - ninv)], θinit) for i in 1:max(draw_samples - draw_samples ÷ 10, draw_samples - 1000)] diff --git a/test/BPINN_PDE_tests.jl b/test/BPINN_PDE_tests.jl index 7923a14daf..f5ef2e7706 100644 --- a/test/BPINN_PDE_tests.jl +++ b/test/BPINN_PDE_tests.jl @@ -68,7 +68,7 @@ end bcstd = [0.1], phystd = [0.05], priorsNNw = (0.0, 10.0), - saveats = [1 / 100.0], progress=true) + saveats = [1 / 100.0]) analytic_sol_func(t) = exp(-(t^2) / 2) / (1 + t + t^3) + t^2 ts = sol1.timepoints[1] From 908cb5be4d3fc08cd88a43f917c4a7efd39d7e8c Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 29 Feb 2024 02:33:41 +0530 Subject: [PATCH 29/53] update BPINN_PDE_tests.jl --- test/BPINN_PDE_tests.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/BPINN_PDE_tests.jl b/test/BPINN_PDE_tests.jl index f5ef2e7706..325ebc6f99 100644 --- a/test/BPINN_PDE_tests.jl +++ b/test/BPINN_PDE_tests.jl @@ -36,7 +36,7 @@ Random.seed!(100) u_real = [analytic_sol_func(0.0, t) for t in ts] u_predict = pmean(sol1.ensemblesol[1]) - @test u_predict≈u_real atol=0.05 + @test u_predict≈u_real atol=0.08 @test mean(u_predict .- u_real) < 0.001 end @@ -199,11 +199,11 @@ end bcstd = [0.1], phystd = [0.05], priorsNNw = (0.0, 10.0), - saveats = [1 / 100.0]) + saveats = [1 / 100.0],progress=true) analytic_sol_func(t) = exp(-(t^2) / 2) / (1 + t + t^3) + t^2 ts = sol1.timepoints[1] u_real = vec([analytic_sol_func(t) for t in ts]) u_predict = pmean(sol1.ensemblesol[1]) - @test u_predict≈u_real atol=0.1 + @test u_predict≈u_real atol=0.5 end From 585a4f5717deaae48b124eb9ebdf07d3c03d7562 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 29 Feb 2024 10:33:18 +0530 Subject: [PATCH 30/53] update BPINN_PDE_tests.jl --- test/BPINN_PDE_tests.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/BPINN_PDE_tests.jl b/test/BPINN_PDE_tests.jl index 325ebc6f99..827b75cd6e 100644 --- a/test/BPINN_PDE_tests.jl +++ b/test/BPINN_PDE_tests.jl @@ -25,10 +25,10 @@ Random.seed!(100) sol1 = ahmc_bayesian_pinn_pde(pde_system, discretization; - draw_samples = 1500, + draw_samples = 2000, bcstd = [0.02], phystd = [0.01], - priorsNNw = (0.0, 1.0), + priorsNNw = (0.0, 10.0), saveats = [1 / 50.0]) analytic_sol_func(u0, t) = u0 + sin(2 * π * t) / (2 * π) @@ -36,8 +36,8 @@ Random.seed!(100) u_real = [analytic_sol_func(0.0, t) for t in ts] u_predict = pmean(sol1.ensemblesol[1]) - @test u_predict≈u_real atol=0.08 - @test mean(u_predict .- u_real) < 0.001 + @test u_predict≈u_real atol=0.05 + @test mean(u_predict .- u_real) < 1e-5 end @testset "Example 2: 1D ODE" begin @@ -199,7 +199,7 @@ end bcstd = [0.1], phystd = [0.05], priorsNNw = (0.0, 10.0), - saveats = [1 / 100.0],progress=true) + saveats = [1 / 100.0]) analytic_sol_func(t) = exp(-(t^2) / 2) / (1 + t + t^3) + t^2 ts = sol1.timepoints[1] From cf77408d7f80e0163d2fc1195b5508d8bbdf7058 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Tue, 26 Mar 2024 19:54:59 +0530 Subject: [PATCH 31/53] done for now --- src/BPINN_ode.jl | 12 +++-- src/PDE_BPINN.jl | 4 +- test/BPINN_PDE_tests.jl | 2 +- test/BPINN_PDEinvsol_tests.jl | 94 ++++++++++++++++++++++++++++++++++- test/bpinnexperimental.jl | 82 +++++++++++++++++++----------- 5 files changed, 156 insertions(+), 38 deletions(-) diff --git a/src/BPINN_ode.jl b/src/BPINN_ode.jl index f092d043c5..e1567bf4e2 100644 --- a/src/BPINN_ode.jl +++ b/src/BPINN_ode.jl @@ -100,6 +100,8 @@ struct BNNODE{C, K, IT <: NamedTuple, init_params::I Adaptorkwargs::A Integratorkwargs::IT + numensemble::Int64 + estim_collocate::Bool autodiff::Bool progress::Bool verbose::Bool @@ -112,6 +114,8 @@ function BNNODE(chain, Kernel = HMC; strategy = nothing, draw_samples = 2000, Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), Integratorkwargs = (Integrator = Leapfrog,), + numensemble = floor(Int, alg.draw_samples / 3), + estim_collocate = false, autodiff = false, progress = false, verbose = false) !(chain isa Lux.AbstractExplicitLayer) && (chain = Lux.transform(chain)) BNNODE(chain, Kernel, strategy, @@ -119,6 +123,7 @@ function BNNODE(chain, Kernel = HMC; strategy = nothing, draw_samples = 2000, phystd, dataset, physdt, MCMCkwargs, nchains, init_params, Adaptorkwargs, Integratorkwargs, + numensemble, estim_collocate, autodiff, progress, verbose) end @@ -177,13 +182,12 @@ function DiffEqBase.__solve(prob::DiffEqBase.ODEProblem, reltol = 1.0f-3, verbose = false, saveat = 1 / 50.0, - maxiters = nothing, - numensemble = floor(Int, alg.draw_samples / 3), - estim_collocate = false) + maxiters = nothing) + @unpack chain, l2std, phystd, param, priorsNNw, Kernel, strategy, draw_samples, dataset, init_params, nchains, physdt, Adaptorkwargs, Integratorkwargs, - MCMCkwargs, autodiff, progress, verbose = alg + MCMCkwargs, numensemble, estim_collocate, autodiff, progress, verbose = alg # ahmc_bayesian_pinn_ode needs param=[] for easier vcat operation for full vector of parameters param = param === nothing ? [] : param diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index b18c35aa99..516ecdcb97 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -394,7 +394,7 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; datafree_colloc_loss_functions[i], nothing; train_sets_pde = colloc_train_sets[i], - train_sets_bc = nothing) + train_sets_bc = nothing)[1] for i in eachindex(datafree_colloc_loss_functions)] function L2_loss2(θ, allstd) @@ -402,7 +402,7 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; # first vector of losses,from tuple -> pde losses, first[1] pde loss pde_loglikelihoods = [sum([pde_loss_function(θ, stdpdesnew[i]) - for (i, pde_loss_function) in enumerate(pde_loss_functions[1])]) + for (i, pde_loss_function) in enumerate(pde_loss_functions)]) for pde_loss_functions in pde_loss_function_points] # bc_loglikelihoods = [sum([bc_loss_function(θ, stdpdesnew[i]) for (i, bc_loss_function) in enumerate(pde_loss_function_points[1])]) for pde_loss_function_points in pde_loss_functions] diff --git a/test/BPINN_PDE_tests.jl b/test/BPINN_PDE_tests.jl index 827b75cd6e..98edd13fa0 100644 --- a/test/BPINN_PDE_tests.jl +++ b/test/BPINN_PDE_tests.jl @@ -37,7 +37,7 @@ Random.seed!(100) u_predict = pmean(sol1.ensemblesol[1]) @test u_predict≈u_real atol=0.05 - @test mean(u_predict .- u_real) < 1e-5 + @test mean(u_predict .- u_real) < 1e-3 end @testset "Example 2: 1D ODE" begin diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 837a5cc79b..7e2356c5bc 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -152,4 +152,96 @@ end p_ = sol1.estimated_de_params[1] @test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] # @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] -end \ No newline at end of file +end + +function recur_expression(exp, Dict_differentials) + for in_exp in exp.args + if !(in_exp isa Expr) + # skip +,== symbols, characters etc + continue + + elseif in_exp.args[1] isa ModelingToolkit.Differential + # first symbol of differential term + # Dict_differentials for masking differential terms + # and resubstituting differentials in equations after putting in interpolations + # temp = in_exp.args[end] + Dict_differentials[eval(in_exp)] = Symbolics.variable("diff_$(length(Dict_differentials) + 1)") + return + else + recur_expression(in_exp, Dict_differentials) + end + end +end + +println("Example 3: 2D Periodic System with New parameter estimation") +@parameters t, p +@variables u(..) + +Dt = Differential(t) +eqs = Dt(u(t)) - cos(p * t) * u(t) ~ 0 +bcs = [u(0) ~ 0.0] +domains = [t ∈ Interval(0.0, 2.0)] + +chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) +initl, st = Lux.setup(Random.default_rng(), chainl) + +@named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [u(t)], + [p], + defaults = Dict([p => 4.0])) + +analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) +timepoints = collect(0.0:(1 / 100.0):2.0) +u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] +u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) +dataset = [hcat(u1, timepoints)] + +discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, + dataset = [dataset, nothing]) + +# creating dictionary for masking equations +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], phystdnew = [0.05], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(6.0, 0.5)], + Dict_differentials = Dict_differentials, + progress = true) + +sol2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(6.0, 0.5)], + progress = true) + +param = 2 * π +ts = vec(sol1.timepoints[1]) +u_real = [analytic_sol_func1(0.0, t) for t in ts] +u_predict = pmean(sol1.ensemblesol[1]) + +@test u_predict≈u_real atol=1.5 +@test mean(u_predict .- u_real) < 0.1 +@test sol1.estimated_de_params[1]≈param atol=param * 0.3 + +ts = vec(sol2.timepoints[1]) +u_real = [analytic_sol_func1(0.0, t) for t in ts] +u_predict = pmean(sol2.ensemblesol[1]) + +@test u_predict≈u_real atol=1.5 +@test mean(u_predict .- u_real) < 0.1 +@test sol1.estimated_de_params[1]≈param atol=param * 0.3 \ No newline at end of file diff --git a/test/bpinnexperimental.jl b/test/bpinnexperimental.jl index 3de049bf58..a8a389ad44 100644 --- a/test/bpinnexperimental.jl +++ b/test/bpinnexperimental.jl @@ -44,20 +44,32 @@ plot!(solution, labels = ["x" "y"]) chain = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), Lux.Dense(6, 2)) -alg = BNNODE(chain; -dataset = dataset, -draw_samples = 1000, -l2std = [0.1, 0.1], -phystd = [0.1, 0.1], -priorsNNw = (0.0, 3.0), -param = [ - Normal(1, 2), - Normal(2, 2), - Normal(2, 2), - Normal(0, 2)], progress = true) - -@time sol_pestim1 = solve(prob, alg; saveat = dt,) -@time sol_pestim2 = solve(prob, alg; estim_collocate = true, saveat = dt) +alg1 = BNNODE(chain; + dataset = dataset, + draw_samples = 1000, + l2std = [0.1, 0.1], + phystd = [0.1, 0.1], + priorsNNw = (0.0, 3.0), + param = [ + Normal(1, 2), + Normal(2, 2), + Normal(2, 2), + Normal(0, 2)], progress = true) + +alg2 = BNNODE(chain; + dataset = dataset, + draw_samples = 1000, + l2std = [0.1, 0.1], + phystd = [0.1, 0.1], + priorsNNw = (0.0, 3.0), + param = [ + Normal(1, 2), + Normal(2, 2), + Normal(2, 2), + Normal(0, 2)], estim_collocate = true, progress = true) + +@time sol_pestim1 = solve(prob, alg1; saveat = dt) +@time sol_pestim2 = solve(prob, alg2; saveat = dt) plot(times, sol_pestim1.ensemblesol[1], label = "estimated x1") plot!(times, sol_pestim2.ensemblesol[1], label = "estimated x2") plot!(times, sol_pestim1.ensemblesol[2], label = "estimated y1") @@ -66,28 +78,29 @@ plot!(times, sol_pestim2.ensemblesol[2], label = "estimated y2") # comparing it with the original solution plot!(solution, labels = ["true x" "true y"]) -@show sol_pestim1.estimated_ode_params -@show sol_pestim2.estimated_ode_params +@show sol_pestim1.estimated_de_params +@show sol_pestim2.estimated_de_params -function fitz(u, p , t) +function fitz(u, p, t) v, w = u[1], u[2] - a,b,τinv,l = p[1], p[2], p[3], p[4] - - dv = v - 0.33*v^3 -w + l - dw = τinv*(v + a - b*w) + a, b, τinv, l = p[1], p[2], p[3], p[4] + + dv = v - 0.33 * v^3 - w + l + dw = τinv * (v + a - b * w) return [dv, dw] end -prob_ode_fitzhughnagumo = ODEProblem(fitz, [1.0,1.0], (0.0,10.0), [0.7,0.8,1/12.5,0.5]) +prob_ode_fitzhughnagumo = ODEProblem( + fitz, [1.0, 1.0], (0.0, 10.0), [0.7, 0.8, 1 / 12.5, 0.5]) dt = 0.5 sol = solve(prob_ode_fitzhughnagumo, Tsit5(), saveat = dt) sig = 0.20 data = Array(sol) -dataset = [data[1,:] .+ (sig .* rand(length(sol.t))), data[2, :] .+ (sig .* rand(length(sol.t))), sol.t] -priors = [Normal(0.5,1.0), Normal(0.5,1.0), Normal(0.0,0.5), Normal(0.5,1.0)] - +dataset = [data[1, :] .+ (sig .* rand(length(sol.t))), + data[2, :] .+ (sig .* rand(length(sol.t))), sol.t] +priors = [Normal(0.5, 1.0), Normal(0.5, 1.0), Normal(0.0, 0.5), Normal(0.5, 1.0)] plot(sol.t, dataset[1], label = "noisy x") plot!(sol.t, dataset[2], label = "noisy y") @@ -98,7 +111,7 @@ chain = Lux.Chain(Lux.Dense(1, 10, tanh), Lux.Dense(10, 10, tanh), Adaptorkwargs = (Adaptor = AdvancedHMC.StanHMCAdaptor, Metric = AdvancedHMC.DiagEuclideanMetric, targetacceptancerate = 0.8) -alg = BNNODE(chain; +alg1 = BNNODE(chain; dataset = dataset, draw_samples = 1000, l2std = [0.1, 0.1], @@ -107,12 +120,21 @@ priorsNNw = (0.01, 3.0), Adaptorkwargs = Adaptorkwargs, param = priors, progress = true) -@time sol_pestim3 = solve(prob_ode_fitzhughnagumo, alg; saveat = dt) -@time sol_pestim4 = solve(prob_ode_fitzhughnagumo, alg; estim_collocate = true, saveat = dt) +alg2 = BNNODE(chain; + dataset = dataset, + draw_samples = 1000, + l2std = [0.1, 0.1], + phystd = [0.1, 0.1], + priorsNNw = (0.01, 3.0), + Adaptorkwargs = Adaptorkwargs, + param = priors, estim_collocate = true, progress = true) + +@time sol_pestim3 = solve(prob_ode_fitzhughnagumo, alg1; saveat = dt) +@time sol_pestim4 = solve(prob_ode_fitzhughnagumo, alg2; saveat = dt) plot!(sol.t, sol_pestim3.ensemblesol[1], label = "estimated x1") plot!(sol.t, sol_pestim4.ensemblesol[1], label = "estimated x2") plot!(sol.t, sol_pestim3.ensemblesol[2], label = "estimated y1") plot!(sol.t, sol_pestim4.ensemblesol[2], label = "estimated y2") -@show sol_pestim3.estimated_ode_params -@show sol_pestim4.estimated_ode_params +@show sol_pestim3.estimated_de_params +@show sol_pestim4.estimated_de_params From 16e1b56d5243f0ff6c68414407c5f9b94043b42f Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Tue, 26 Mar 2024 20:50:40 +0530 Subject: [PATCH 32/53] merge conflict resolution --- src/NeuralPDE.jl | 2 ++ src/advancedHMC_MCMC.jl | 20 +++++++------------- src/collocated_estim.jl | 27 --------------------------- 3 files changed, 9 insertions(+), 40 deletions(-) diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 9bbdf5ce49..4799cf0bae 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -30,6 +30,7 @@ using DomainSets: Domain, ClosedInterval, AbstractInterval, leftendpoint, righte using SciMLBase: @add_kwonly, parameterless_type using UnPack: @unpack import ChainRulesCore, Lux, ComponentArrays +using Lux: FromFluxAdaptor using ChainRulesCore: @non_differentiable RuntimeGeneratedFunctions.init(@__MODULE__) @@ -51,6 +52,7 @@ include("neural_adapter.jl") include("advancedHMC_MCMC.jl") include("BPINN_ode.jl") include("PDE_BPINN.jl") +include("dgm.jl") include("collocated_estim.jl") export NNODE, NNDAE, diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index 9064ddd9fa..7ca18bd58b 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -86,14 +86,6 @@ end vector_to_parameters(ps_new::AbstractVector, ps::AbstractVector) = ps_new function LogDensityProblems.logdensity(Tar::LogTargetDensity, θ) - # if Tar.estim_collocate - # return physloglikelihood(Tar, θ) / length(Tar.dataset[1]) + priorweights(Tar, θ) + - # L2LossData(Tar, θ) / length(Tar.dataset[1]) + - # L2loss2(Tar, θ) / length(Tar.dataset[1]) - # else - # return physloglikelihood(Tar, θ) / length(Tar.dataset[1]) + priorweights(Tar, θ) + - # L2LossData(Tar, θ) / length(Tar.dataset[1]) - # end if Tar.estim_collocate return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) + L2loss2(Tar, θ) @@ -455,7 +447,9 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; MCMCkwargs = (n_leapfrog = 30,), progress = false, verbose = false, estim_collocate = false) - !(chain isa Lux.AbstractExplicitLayer) && (chain = Lux.transform(chain)) + + !(chain isa Lux.AbstractExplicitLayer) && + (chain = adapt(FromFluxAdaptor(false, false), chain)) # NN parameter prior mean and variance(PriorsNN must be a tuple) if isinplace(prob) throw(error("The BPINN ODE solver only supports out-of-place ODE definitions, i.e. du=f(u,p,t).")) @@ -464,7 +458,7 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; strategy = strategy == GridTraining ? strategy(physdt) : strategy if dataset != [nothing] && - (length(dataset) < 2 || !(typeof(dataset) <: Vector{<:Vector{<:AbstractFloat}})) + (length(dataset) < 2 || !(dataset isa Vector{<:Vector{<:AbstractFloat}})) throw(error("Invalid dataset. dataset would be timeseries (x̂,t) where type: Vector{Vector{AbstractFloat}")) end @@ -475,7 +469,7 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; end if chain isa Lux.AbstractExplicitLayer - # Flux-vector, Lux-Named Tuple + # Lux-Named Tuple initial_nnθ, recon, st = generate_Tar(chain, init_params) else error("Only Lux.AbstractExplicitLayer Neural networks are supported") @@ -553,7 +547,7 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; MCMC_alg = kernelchoice(Kernel, MCMCkwargs) Kernel = AdvancedHMC.make_kernel(MCMC_alg, integrator) samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, adaptor; - progress = progress, verbose = verbose, drop_warmup = true) + progress = progress, verbose = verbose) samplesc[i] = samples statsc[i] = stats @@ -571,7 +565,7 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; MCMC_alg = kernelchoice(Kernel, MCMCkwargs) Kernel = AdvancedHMC.make_kernel(MCMC_alg, integrator) samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, - adaptor; progress = progress, verbose = verbose, drop_warmup = true) + adaptor; progress = progress, verbose = verbose) @info("Sampling Complete.") @info("Current Physics Log-likelihood : ", physloglikelihood(ℓπ, samples[end])) diff --git a/src/collocated_estim.jl b/src/collocated_estim.jl index a2f81b3ed9..3902f74a27 100644 --- a/src/collocated_estim.jl +++ b/src/collocated_estim.jl @@ -162,31 +162,4 @@ function calderivatives(prob, dataset) gradients = (A2 .- A1) ./ sqrt(eps(eltype(dataset[end][1]))) return gradients -end - -function calculate_derivatives(dataset) - - # u = dataset[1] - # u1 = dataset[2] - # t = dataset[end] - # # control points - # n = Int(floor(length(t) / 10)) - # # spline for datasetvalues(solution) - # # interp = BSplineApprox(u, t, 4, 10, :Uniform, :Uniform) - # interp = CubicSpline(u, t) - # interp1 = CubicSpline(u1, t) - # # derivatives interpolation - # dx = t[2] - t[1] - # time = collect(t[1]:dx:t[end]) - # smoothu = [interp(i) for i in time] - # smoothu1 = [interp1(i) for i in time] - # # derivative of the spline (must match function derivative) - # û = tvdiff(smoothu, 20, 0.5, dx = dx, ε = 1) - # û1 = tvdiff(smoothu1, 20, 0.5, dx = dx, ε = 1) - # # tvdiff(smoothu, 100, 0.035, dx = dx, ε = 1) - # # FDM - # # û1 = diff(u) / dx - # # dataset[1] and smoothu are almost equal(rounding errors) - # return [û, û1] - end \ No newline at end of file From 9f694f855a3d64cc314c868be76953694d744fb8 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Tue, 26 Mar 2024 20:54:21 +0530 Subject: [PATCH 33/53] update NeuralPDE.jl, advancedHMC_MCMC.jl --- src/NeuralPDE.jl | 1 - src/advancedHMC_MCMC.jl | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 4799cf0bae..890f938309 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -53,7 +53,6 @@ include("advancedHMC_MCMC.jl") include("BPINN_ode.jl") include("PDE_BPINN.jl") include("dgm.jl") -include("collocated_estim.jl") export NNODE, NNDAE, PhysicsInformedNN, discretize, diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index 7ca18bd58b..fdcdbdfcb4 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -448,8 +448,7 @@ function ahmc_bayesian_pinn_ode(prob::DiffEqBase.ODEProblem, chain; progress = false, verbose = false, estim_collocate = false) - !(chain isa Lux.AbstractExplicitLayer) && - (chain = adapt(FromFluxAdaptor(false, false), chain)) + !(chain isa Lux.AbstractExplicitLayer) && (chain = adapt(FromFluxAdaptor(false, false), chain)) # NN parameter prior mean and variance(PriorsNN must be a tuple) if isinplace(prob) throw(error("The BPINN ODE solver only supports out-of-place ODE definitions, i.e. du=f(u,p,t).")) From a28d12f065a899ebff0eb9f36b833d1559ad947e Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Tue, 26 Mar 2024 20:55:24 +0530 Subject: [PATCH 34/53] update NeuralPDE.jl --- src/NeuralPDE.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 890f938309..4799cf0bae 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -53,6 +53,7 @@ include("advancedHMC_MCMC.jl") include("BPINN_ode.jl") include("PDE_BPINN.jl") include("dgm.jl") +include("collocated_estim.jl") export NNODE, NNDAE, PhysicsInformedNN, discretize, From 03ee7b48b0c5f241bd2b59e1ec28181218783182 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Tue, 26 Mar 2024 21:00:38 +0530 Subject: [PATCH 35/53] update NeuralPDE.jl --- src/NeuralPDE.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 4799cf0bae..890f938309 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -53,7 +53,6 @@ include("advancedHMC_MCMC.jl") include("BPINN_ode.jl") include("PDE_BPINN.jl") include("dgm.jl") -include("collocated_estim.jl") export NNODE, NNDAE, PhysicsInformedNN, discretize, From 27cfb56fb19fc487d4d4964da9dce69eb0376980 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Tue, 26 Mar 2024 21:01:57 +0530 Subject: [PATCH 36/53] update NeuralPDE.jl --- src/NeuralPDE.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 890f938309..4799cf0bae 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -53,6 +53,7 @@ include("advancedHMC_MCMC.jl") include("BPINN_ode.jl") include("PDE_BPINN.jl") include("dgm.jl") +include("collocated_estim.jl") export NNODE, NNDAE, PhysicsInformedNN, discretize, From 6ef9d48a8bbd9313902e5fea0359e07ecc0ddb50 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Fri, 29 Mar 2024 22:02:18 +0530 Subject: [PATCH 37/53] pmean for tests --- src/BPINN_ode.jl | 2 +- test/BPINN_PDEinvsol_tests.jl | 6 ++---- test/BPINN_Tests.jl | 8 ++++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/BPINN_ode.jl b/src/BPINN_ode.jl index 756696826d..ba699ee035 100644 --- a/src/BPINN_ode.jl +++ b/src/BPINN_ode.jl @@ -114,7 +114,7 @@ function BNNODE(chain, Kernel = HMC; strategy = nothing, draw_samples = 2000, Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), Integratorkwargs = (Integrator = Leapfrog,), - numensemble = floor(Int, alg.draw_samples / 3), + numensemble = floor(Int, draw_samples / 3), estim_collocate = false, autodiff = false, progress = false, verbose = false) !(chain isa Lux.AbstractExplicitLayer) && (chain = adapt(FromFluxAdaptor(false, false), chain)) diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 7e2356c5bc..748ff21686 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -216,8 +216,7 @@ sol1 = ahmc_bayesian_pinn_pde(pde_system, priorsNNw = (0.0, 1.0), saveats = [1 / 50.0], param = [LogNormal(6.0, 0.5)], - Dict_differentials = Dict_differentials, - progress = true) + Dict_differentials = Dict_differentials) sol2 = ahmc_bayesian_pinn_pde(pde_system, discretization; @@ -226,8 +225,7 @@ sol2 = ahmc_bayesian_pinn_pde(pde_system, phystd = [0.01], l2std = [0.01], priorsNNw = (0.0, 1.0), saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)], - progress = true) + param = [LogNormal(6.0, 0.5)]) param = 2 * π ts = vec(sol1.timepoints[1]) diff --git a/test/BPINN_Tests.jl b/test/BPINN_Tests.jl index 2fe347b3b4..6a32c560f0 100644 --- a/test/BPINN_Tests.jl +++ b/test/BPINN_Tests.jl @@ -53,8 +53,8 @@ Random.seed!(100) @test mean(abs.(physsol1 .- meanscurve)) < 0.005 #--------------------- solve() call - @test mean(abs.(x̂1 .- sol1lux.ensemblesol[1])) < 0.05 - @test mean(abs.(physsol0_1 .- sol1lux.ensemblesol[1])) < 0.05 + @test mean(abs.(x̂1 .- pmean(sol1lux.ensemblesol[1]))) < 0.025 + @test mean(abs.(physsol0_1 .- pmean(sol1lux.ensemblesol[1]))) < 0.025 end @testset "Example 2 - with parameter estimation" begin @@ -122,7 +122,7 @@ end @test abs(p - mean([fhsamples[i][23] for i in 2000:length(fhsamples)])) < abs(0.35 * p) #-------------------------- solve() call - @test mean(abs.(physsol1_1 .- sol2lux.ensemblesol[1])) < 8e-2 + @test mean(abs.(physsol1_1 .- pmean(sol2lux.ensemblesol[1]))) < 8e-2 # ESTIMATED ODE PARAMETERS (NN1 AND NN2) @test abs(p - sol2lux.estimated_de_params[1]) < abs(0.15 * p) @@ -211,7 +211,7 @@ end #-------------------------- solve() call # (lux chain) - @test mean(abs.(physsol2 .- sol3lux_pestim.ensemblesol[1])) < 0.15 + @test mean(abs.(physsol2 .- pmean(sol3lux_pestim.ensemblesol[1]))) < 0.15 # estimated parameters(lux chain) param1 = sol3lux_pestim.estimated_de_params[1] @test abs(param1 - p) < abs(0.45 * p) From f8cf2da13eb044c210b2550f9db32097a44cac13 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Fri, 29 Mar 2024 22:11:25 +0530 Subject: [PATCH 38/53] . --- test/BPINN_PDEinvsol_tests.jl | 1113 ++++++++++++++++++++++++++++++++- 1 file changed, 1112 insertions(+), 1 deletion(-) diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 748ff21686..1fadc8a0a5 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -242,4 +242,1115 @@ u_predict = pmean(sol2.ensemblesol[1]) @test u_predict≈u_real atol=1.5 @test mean(u_predict .- u_real) < 0.1 -@test sol1.estimated_de_params[1]≈param atol=param * 0.3 \ No newline at end of file +@test sol1.estimated_de_params[1]≈param atol=param * 0.3 + + + +println("Example 3: Lotka Volterra with New parameter estimation") +@parameters t α β γ δ +@variables x(..) y(..) + +Dt = Differential(t) +eqs = [Dt(x(t))*α ~ x(t) - β * x(t) * y(t), Dt(y(t))*γ ~ δ * x(t) * y(t) - y(t)] +bcs = [x(0) ~ 1.0, y(0) ~ 1.0] +domains = [t ∈ Interval(0.0, 7.0)] + +# Define the parameters' values +# params = [α => 1.0, β => 0.5, γ => 0.5, δ => 1.0] +# p = [1.5, 1.0, 3.0, 1.0] + +chainl = [ + Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh),Lux.Dense(6, 1)), + Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh),Lux.Dense(6, 1)) +] + +initl, st = Lux.setup(Random.default_rng(), chainl[1]) +initl1, st1 = Lux.setup(Random.default_rng(), chainl[2]) + +using NeuralPDE, Lux, OrdinaryDiffEq, Distributions, Random + +function lotka_volterra(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end +# initial-value problem. +u0 = [1.0, 1.0] +# p = [2/3, 2/3, 1/3.0, 1/3.0] +p = [1.5, 1.0, 3.0, 1.0] +tspan = (0.0, 7.0) +prob = ODEProblem(lotka_volterra, u0, tspan, p) +dt = 0.01 +solution = solve(prob, Tsit5(); saveat = dt) + + +# function moving_average_smoothing(data::Vector{T}, window_size::Int) where {T} +# smoothed_data = similar(data, T, length(data)) + +# for i in 1:length(data) +# start_idx = max(1, i - window_size) +# end_idx = min(length(data), i + window_size) +# smoothed_data[i] = mean(data[start_idx:end_idx]) +# end + +# return smoothed_data' +# end + +# Extract solution +time = solution.t +u = hcat(solution.u...) +time1=solution.t +u_noisy = u .+ u .* (0.3 .* randn(size(u))) + +plot(time,u[1,:]) +plot!(time,u[2,:]) +scatter!(time1,u_noisy[1,:]) +scatter!(time1,u_noisy[2,:]) + +# window_size = 5 +# smoothed_datasets = [moving_average_smoothing(u1[i, :], window_size) +# for i in 1:length(solution.u[1])] +# u2 = vcat(smoothed_datasets[1], smoothed_datasets[2]) +# Randomly select some points from the solution +num_points = 150 # Number of points to select +selected_indices = rand(1:size(u_noisy, 2), num_points) +upoints = [u_noisy[:, i] for i in selected_indices] +timepoints = [time[i] for i in selected_indices] +temp=hcat(upoints...) +dataset = [hcat(temp[i, :], timepoints) for i in 1:2] + +# plot(time,u[1,:]) +# plot!(time,u[2,:]) + +discretization = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, + dataset = [dataset, nothing]) + +@named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [x(t), y(t)], + [α, β, γ, δ], + defaults = Dict([α =>2, β => 3, γ =>3, δ =>2])) + +# creating dictionary for masking equations +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol3 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], progress = true) + +# time +# dataset +# chainl[1](time', sol3.estimated_nn_params[1], st)[1][1,:] +# plot!(time1, chainl[1](time1', sol3.estimated_nn_params[1], st)[1][1,:]) +# plot!(time1, chainl[2](time1', sol3.estimated_nn_params[2], st)[1][1,:]) +# plot!(time1, chainl[1](time1', sol5.estimated_nn_params[1], st)[1][1,:]) +# plot!(time1, chainl[2](time1', sol5.estimated_nn_params[2], st)[1][1,:]) +# time1 = collect(0.0:(1 / 100.0):8.0) + +sol4 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], progress = true +) + + +sol5_00 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.15, 0.15], + phystd = [0.15, 0.15], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5_0 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.05, 0.05], + phystd = [0.05, 0.05], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.1, 0.1], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +# 100 points(sol5_2 vs sol3) +sol5_2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.1, 0.1], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +# 100 points(sol5_2 vs sol3) +sol5_2_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.08, 0.08], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +# 100 points(sol5_2 vs sol3) +sol5_2_2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +# 50 datapoint 0-5 sol5 vs sol4 +# julia> sol4.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.549 ± 0.0058 +# 0.71 ± 0.0042 +# 0.408 ± 0.0063 +# 0.355 ± 0.0015 + +# julia> sol5.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.604 ± 0.0052 +# 0.702 ± 0.0034 +# 0.346 ± 0.0037 +# 0.335 ± 0.0013 + +# 100 datapoint 0-5 sol5_2 vs sol3 +# julia> sol3.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.598 ± 0.0037 +# 0.711 ± 0.0027 +# 0.399 ± 0.0032 +# 0.333 ± 0.0011 + +# julia> sol5_2.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.604 ± 0.0035 +# 0.686 ± 0.0026 +# 0.395 ± 0.0029 +# 0.328 ± 0.00095 + +# timespan for full dataset (0-8) +sol6 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], progress = true) + +sol5_3 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5_4 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5_5 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], progress = true +) + +sol7 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol5_5_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], progress = true +) + +sol7_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol7_2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.1, 0.1], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol7_3 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol7_4 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +lpfun = function f(chain::Chains) # function to compute the logpdf values + niter, nparams, nchains = size(chain) + lp = zeros(niter + nchains) # resulting logpdf values + for i = 1:nparams + lp += logpdf(MvNormal(Array(chain[:,i,:])) , dataset[1][:,1]') + lp += logpdf(MvNormal(Array(chain[:,i,:])) , dataset[1][:,2]') + end + return lp +end + +DIC, pD = dic(sol3.original.mcmc_chain, lpfun) +DIC1, pD1 = dic(sol4.original.mcmc_chain, lpfun) + +size(sol3.original.mcmc_chain) +Array(sol3.original.mcmc_chain[1,:,:]) +length(sol3.estimated_nn_params[1]) +chainl[1](time', sol3.estimated_nn_params[1], st)[1] + +data = [hcat(calculate_derivatives2(dataset[i][:, 2], dataset[1][:, 1]),dataset[i][:, 2]) for i in eachindex(dataset)] +dataset[1][:,1] +dataset[2] +plot!(dataset[1][:,2],dataset[1][:,1]) +eqs +sol5 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 200, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.02, 0.02], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(3, 2), + Normal(3, 2) + # LogNormal(1, 2), + # LogNormal(1, 2), + # LogNormal(1, 2), + # LogNormal(1, 2) + ], progress = true) + +# plot(time, chainl[1](time', sol2.estimated_nn_params[1], st)[1]) +# plot!(time, chainl[2](time', sol2.estimated_nn_params[2], st)[1]) + +sol6 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 200, + bcstd = [0.5, 0.5], + phystd = [0.5, 0.5], l2std = [0.02, 0.02], + priorsNNw = (0.0, 5.0), phystdnew = [0.5, 0.5], + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8),aa + param = [ + # LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3) + # Normal(3, 2), + # Normal(4, 2), + Normal(3, 2), + Normal(3, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +function calculate_derivatives2(indvar,depvar) + x̂, time = indvar,depvar + num_points = length(x̂) + # Initialize an array to store the derivative values. + derivatives = similar(x̂) + + for i in 2:(num_points - 1) + # Calculate the first-order derivative using central differences. + Δt_forward = time[i + 1] - time[i] + Δt_backward = time[i] - time[i - 1] + + derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) + + derivatives[i] = derivative + end + + # Derivatives at the endpoints can be calculated using forward or backward differences. + derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) + derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) + return derivatives +end +dataset[1] +dataset[2] +dataset[1][:,1]=calculate_derivatives2(dataset[1][:,2], dataset[1][:,1]) +dataset[2][:,1]=calculate_derivatives2(dataset[2][:,2], dataset[2][:,1]) +dataset[1] +dataset[2] +sol7 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 200, + bcstd = [0.5, 0.5], + phystd = [0.5, 0.5], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(0, 2), + Normal(0, 2) + # LogNormal(1, 2), + # LogNormal(1, 2), + # LogNormal(1, 2), + # LogNormal(1, 2) + ], progress = true) + +# plot(time, chainl[1](time', sol2.estimated_nn_params[1], st)[1]) +# plot!(time, chainl[2](time', sol2.estimated_nn_params[2], st)[1]) + +sol8 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), phystdnew = [0.1, 0.1], + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8),aa + param = [ + # LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3) + # Normal(3, 2), + # Normal(4, 2), + Normal(0, 2), + Normal(0, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +timepoints = collect(0.0:(1 / 100.0):9.0) +plot!(timepoints', chainl[1](timepoints', sol5_4.estimated_nn_params[1], st)[1]) +plot!(timepoints, chainl[2](timepoints', sol5_4.estimated_nn_params[2], st)[1]) + +using Plots, StatsPlots +plotly() + +plot(time, u[1, :]) +plot!(time, u[2, :]) +scatter!(time, u_noisy[1, :]) +scatter!(time, u_noisy[2, :]) +scatter!(discretization.dataset[1][1][:,2], discretization.dataset[1][1][:,1]) +scatter!(discretization.dataset[1][2][:,2], discretization.dataset[1][2][:,1]) + +# plot28(sol4 seems better vs sol3 plots, params seems similar) +plot!(sol3.timepoints[1]', sol3.ensemblesol[1],legend=nothing) +plot!(sol3.timepoints[2]', sol3.ensemblesol[2]) +plot!(sol4.timepoints[1]', sol4.ensemblesol[1]) +plot!(sol4.timepoints[2]', sol4.ensemblesol[2]) + +plot!(sol4_2.timepoints[1]', sol4_2.ensemblesol[1],legend=nothing) +plot!(sol4_2.timepoints[2]', sol4_2.ensemblesol[2]) +plot!(sol5_2.timepoints[1]', sol5_2.ensemblesol[1],legend=nothing) +plot!(sol5_2.timepoints[2]', sol5_2.ensemblesol[2]) + +plot!(sol4_3.timepoints[1]', sol4_3.ensemblesol[1],legend=nothing) +plot!(sol4_3.timepoints[2]', sol4_3.ensemblesol[2]) +plot!(sol5_3.timepoints[1]', sol5_3.ensemblesol[1]) +plot!(sol5_3.timepoints[2]', sol5_3.ensemblesol[2]) +plot!(sol5_4.timepoints[1]', sol5_4.ensemblesol[1],legend=nothing) +plot!(sol5_4.timepoints[2]', sol5_4.ensemblesol[2]) + + +# plot 36 sol4 vs sol5(params sol4 better, but plots sol5 "looks" better),plot 44(sol5 better than sol6 overall) +plot!(sol5.timepoints[1]', sol5.ensemblesol[1],legend=nothing) +plot!(sol5.timepoints[2]', sol5.ensemblesol[2]) +plot!(sol6.timepoints[1]', sol6.ensemblesol[1]) +plot!(sol6.timepoints[2]', sol6.ensemblesol[2]) + +# plot52 sol7 vs sol5(sol5 overall better plots, params?) +plot!(sol7.timepoints[1]', sol7.ensemblesol[1]) +plot!(sol7.timepoints[2]', sol7.ensemblesol[2]) + +# sol8,sol8_2,sol9,sol9_2 bad +plot!(sol8.timepoints[1]', sol8.ensemblesol[1]) +plot!(sol8.timepoints[2]', sol8.ensemblesol[2]) +plot!(sol8_2.timepoints[1]', sol8_2.ensemblesol[1]) +plot!(sol8_2.timepoints[2]', sol8_2.ensemblesol[2]) + +plot!(sol9.timepoints[1]', sol9.ensemblesol[1]) +plot!(sol9.timepoints[2]', sol9.ensemblesol[2]) +plot!(sol9_2.timepoints[1]', sol9_2.ensemblesol[1]) +plot!(sol9_2.timepoints[2]', sol9_2.ensemblesol[2]) + + +plot!(sol5_5.timepoints[1]', sol5_5.ensemblesol[1]) +plot!(sol5_5.timepoints[2]', sol5_5.ensemblesol[2],legend=nothing) + +plot!(sol5_5_1.timepoints[1]', sol5_5_1.ensemblesol[1]) +plot!(sol5_5_1.timepoints[2]', sol5_5_1.ensemblesol[2],legend=nothing) +plot!(sol7_1.timepoints[1]', sol7_1.ensemblesol[1]) +plot!(sol7_1.timepoints[2]', sol7_1.ensemblesol[2]) + +plot!(sol7_4.timepoints[1]', sol7_4.ensemblesol[1]) +plot!(sol7_4.timepoints[2]', sol7_4.ensemblesol[2]) + +plot!(sol5_2_1.timepoints[1]', sol5_2_1.ensemblesol[1],legend=nothing) +plot!(sol5_2_1.timepoints[2]', sol5_2_1.ensemblesol[2]) +plot!(sol5_2_2.timepoints[1]', sol5_2_2.ensemblesol[1],legend=nothing) +plot!(sol5_2_2.timepoints[2]', sol5_2_2.ensemblesol[2]) + +plot!(sol5_0.timepoints[1]', sol5_0.ensemblesol[1]) +plot!(sol5_0.timepoints[2]', sol5_0.ensemblesol[2],legend=nothing) + +plot!(sol5_00.timepoints[1]', sol5_00.ensemblesol[1]) +plot!(sol5_00.timepoints[2]', sol5_00.ensemblesol[2],legend=nothing) + +# test with lower number of points +# test same calls 2 times or more +# consider full range dataset case +# combination of all above + +# run 1 100 iters +sol5.estimated_de_params +sol6.estimated_de_params + +# run 2 200 iters +sol5.estimated_de_params +sol6.estimated_de_params + +# run 2 200 iters +sol3.estimated_de_params +sol4.estimated_de_params + +# p = [2/3, 2/3, 1/3, 1/3] +sol3.estimated_de_params +sol4.estimated_de_params +dataset[1] +eqs +α, β, γ, δ = p +p +# 1.0 +# 0.6666666666666666 +# 1.0 +# 0.33333333333333333 + +1/a +1/c +eqs +using StatsPlots +plotly() +plot(sol3.original.mcmc_chain) +plot(sol4.original.mcmc_chain) + +# 4-element Vector{Particles{Float64, 34}}: +# 1.23 ± 0.022 +# 0.858 ± 0.011 +# 3.04 ± 0.079 +# 1.03 ± 0.024 +# 4-element Vector{Particles{Float64, 34}}: +# 1.2 ± 0.0069 +# 0.835 ± 0.006 +# 3.22 ± 0.01 +# 1.08 ± 0.0053 +# # plot(time', chainl[1](time', sol1.estimated_nn_params[1], st)[1]) +# # plot!(time, chainl[2](time', sol1.estimated_nn_params[2], st)[1]) + +# sol3 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 500, +# bcstd = [0.05, 0.05], +# phystd = [0.005, 0.005], l2std = [0.1, 0.1], +# phystdnew = [0.5, 0.5], +# # Kernel = AdvancedHMC.NUTS(0.8), +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 50.0], +# param = [ +# Normal(0.0, 2), +# Normal(0.0, 2), +# Normal(0.0, 2), +# Normal(0.0, 2) +# ], +# Dict_differentials = Dict_differentials, progress = true) + +# sol = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 500, +# bcstd = [0.05, 0.05], +# phystd = [0.005, 0.005], l2std = [0.1, 0.1], +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 50.0], +# # Kernel = AdvancedHMC.NUTS(0.8), +# param = [ +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2) +# ], progress = true) + +# plot!(sol.timepoints[1]', sol.ensemblesol[1]) +# plot!(sol.timepoints[2]', sol.ensemblesol[2]) + +# sol1 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 500, +# bcstd = [0.05, 0.05], +# phystd = [0.005, 0.005], l2std = [0.1, 0.1], +# phystdnew = [0.5, 0.5], +# # Kernel = AdvancedHMC.NUTS(0.8), +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 50.0], +# param = [ +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2) +# ], +# Dict_differentials = Dict_differentials, progress = true) + +# plot!(sol1.timepoints[1]', sol1.ensemblesol[1]) +# plot!(sol1.timepoints[2]', sol1.ensemblesol[2]) + +sol = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 500, + bcstd = [0.05, 0.05], + phystd = [0.005, 0.005], l2std = [0.1, 0.1], + priorsNNw = (0.0, 10.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2) + ]) + +# plot!(sol.timepoints[1]', sol.ensemblesol[1]) +# plot!(sol.timepoints[2]', sol.ensemblesol[2]) + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 500, + bcstd = [0.05, 0.05], + phystd = [0.005, 0.005], l2std = [0.1, 0.1], + phystdnew = [0.5, 0.5], + # Kernel = AdvancedHMC.NUTS(0.8), + priorsNNw = (0.0, 10.0), + saveats = [1 / 50.0], + param = [ + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2) + ], + Dict_differentials = Dict_differentials) + +param = 2 * π +ts = vec(sol1.timepoints[1]) +u_real = [analytic_sol_func1(0.0, t) for t in ts] +u_predict = pmean(sol1.ensemblesol[1]) + +@test u_predict≈u_real atol=1.5 +@test mean(u_predict .- u_real) < 0.1 +@test sol1.estimated_de_params[1]≈param atol=param * 0.3 + +# points1 = [] +# for eq_arg in eq_args +# a = [] +# # for each (depvar,[indvar1..]) if indvari==indvar (eq_arg) +# for i in eachindex(symbols_input) +# if symbols_input[i][2] == eq_arg +# # include domain points of that depvar +# # each loss equation take domain matrix [points..;points..] +# push!(a, train_sets[i][:, 2:end]') +# end +# end +# # vcat as new row for next equation +# push!(points1, vcat(a...)) +# end +# println(points1 == points) + +# using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC +# import ModelingToolkit: Interval, infimum, supremum, Distributions +# using Plots, MonteCarloMeasurements + +# @parameters x, t, α +# @variables u(..) +# Dt = Differential(t) +# Dx = Differential(x) +# Dx2 = Differential(x)^2 +# Dx3 = Differential(x)^3 +# Dx4 = Differential(x)^4 + +# # α = 1 +# β = 4 +# γ = 1 +# eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 + +# u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +# du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 + +# bcs = [u(x, 0) ~ u_analytic(x, 0), +# u(-10, t) ~ u_analytic(-10, t), +# u(10, t) ~ u_analytic(10, t), +# Dx(u(-10, t)) ~ du(-10, t), +# Dx(u(10, t)) ~ du(10, t)] + +# # Space and time domains +# domains = [x ∈ Interval(-10.0, 10.0), +# t ∈ Interval(0.0, 1.0)] + +# # Discretization +# dx = 0.4; +# dt = 0.2; + +# # Function to compute analytical solution at a specific point (x, t) +# function u_analytic_point(x, t) +# z = -x / 2 + t +# return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +# end + +# # Function to generate the dataset matrix +# function generate_dataset_matrix(domains, dx, dt) +# x_values = -10:dx:10 +# t_values = 0.0:dt:1.0 + +# dataset = [] + +# for t in t_values +# for x in x_values +# u_value = u_analytic_point(x, t) +# push!(dataset, [u_value, x, t]) +# end +# end + +# return vcat([data' for data in dataset]...) +# end + +# datasetpde = [generate_dataset_matrix(domains, dx, dt)] + +# # noise to dataset +# noisydataset = deepcopy(datasetpde) +# noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ +# randn(size(noisydataset[1][:, 1])) .* 5 / 100 .* +# noisydataset[1][:, 1] + +# # plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") +# # plot!(noisydataset[1][:, 2], noisydataset[1][:, 1]) + +# # Neural network +# chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), +# Lux.Dense(8, 8, Lux.tanh), +# Lux.Dense(8, 1)) + +# discretization = NeuralPDE.BayesianPINN([chain], +# GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) + +# @named pde_system = PDESystem(eq, +# bcs, +# domains, +# [x, t], +# [u(x, t)], +# [α], +# defaults = Dict([α => 0.5])) + +# sol1 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 100, +# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], +# phystd = [1.0], l2std = [0.05], param = [Distributions.LogNormal(0.5, 2)], +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 100.0, 1 / 100.0], progress = true) + +# eqs = pde_system.eqs +# Dict_differentials = Dict() +# exps = toexpr.(eqs) +# nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +# sol2 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 100, +# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], +# phystd = [1.0], phystdnew = [0.05], l2std = [0.05], +# param = [Distributions.LogNormal(0.5, 2)], +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 100.0, 1 / 100.0], Dict_differentials = Dict_differentials, +# progress = true) + +# phi = discretization.phi[1] +# xs, ts = [infimum(d.domain):dx:supremum(d.domain) +# for (d, dx) in zip(domains, [dx / 10, dt])] +# u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] +# for t in ts] +# u_real = [[u_analytic(x, t) for x in xs] for t in ts] +# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) +# for x in xs] +# for t in ts] + +# # p1 = plot(xs, u_predict, title = "predict") +# # p2 = plot(xs, u_real, title = "analytic") +# # p3 = plot(xs, diff_u, title = "error") +# # plot(p1, p2, p3) + +# phi = discretization.phi[1] +# xs, ts = [infimum(d.domain):dx:supremum(d.domain) +# for (d, dx) in zip(domains, [dx / 10, dt])] +# u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] +# for t in ts] +# u_real = [[u_analytic(x, t) for x in xs] for t in ts] +# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) +# for x in xs] +# for t in ts] + +# # p1 = plot(xs, u_predict, title = "predict") +# # p2 = plot(xs, u_real, title = "analytic") +# # p3 = plot(xs, diff_u, title = "error") +# # plot(p1, p2, p3) + +@parameters t, p +@variables u(..) + +Dt = Differential(t) +eqs = Dt(u(t)) - cos(p * t) ~ 0 +bcs = [u(0) ~ 0.0] +domains = [t ∈ Interval(0.0, 2.0)] + +chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) +initl, st = Lux.setup(Random.default_rng(), chainl) + +@named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [u(t)], + [p], + defaults = Dict([p => 4.0])) + +analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) +timepoints = collect(0.0:(1 / 100.0):2.0) +u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] +u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) +dataset = [hcat(u1, timepoints)] + +discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, + dataset = [dataset, nothing]) + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(4.0, 2)], progress = true) + +param = 2 * π +ts = vec(sol1.timepoints[1]) +u_real = [analytic_sol_func1(0.0, t) for t in ts] +u_predict = pmean(sol1.ensemblesol[1]) + +@test u_predict≈u_real atol=0.1 +@test mean(u_predict .- u_real) < 0.01 +@test sol1.estimated_de_params[1]≈param atol=0.1 +sol1.estimated_de_params[1] + +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.02], phystdnew = [0.02], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(4.0, 2)], + Dict_differentials = Dict_differentials, + progress = true) + +param = 2 * π +ts_2 = vec(sol2.timepoints[1]) +u_real_2 = [analytic_sol_func1(0.0, t) for t in ts] +u_predict_2 = pmean(sol2.ensemblesol[1]) + +@test u_predict_2≈u_real_2 atol=0.1 +@test mean(u_predict_2 .- u_real_2) < 0.01 +@test sol2.estimated_de_params[1]≈param atol=0.1 +sol2.estimated_de_params[1] + +plot(ts_2, u_predict_2) +plot!(ts_2, u_real_2) + +@parameters t, σ_ +@variables x(..), y(..), z(..) +Dt = Differential(t) +eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), + Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), + Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] + +bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] +domains = [t ∈ Interval(0.0, 1.0)] + +input_ = length(domains) +n = 7 +chain = [ + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)) +] + +#Generate Data +function lorenz!(du, u, p, t) + du[1] = 10.0 * (u[2] - u[1]) + du[2] = u[1] * (28.0 - u[3]) - u[2] + du[3] = u[1] * u[2] - (8 / 3) * u[3] +end + +u0 = [1.0; 0.0; 0.0] +tspan = (0.0, 1.0) +prob = ODEProblem(lorenz!, u0, tspan) +sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) +ts = sol.t +us = hcat(sol.u...) +us = us .+ ((0.05 .* randn(size(us))) .* us) +ts_ = hcat(sol(ts).t...)[1, :] +dataset = [hcat(us[i, :], ts_) for i in 1:3] + +discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, + dataset = [dataset, nothing]) + +@named pde_system = PDESystem(eqs, bcs, domains, + [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 100, + bcstd = [0.3, 0.3, 0.3], + phystd = [0.1, 0.1, 0.1], + l2std = [1, 1, 1], + priorsNNw = (0.0, 1.0), + saveats = [0.01], + param = [Normal(14.0, 2)], progress = true) + +idealp = 10.0 +p_ = sol1.estimated_de_params[1] +@test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] +# @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] + +@parameters x y +@variables u(..) +Dxx = Differential(x)^2 +Dyy = Differential(y)^2 + +# 2D PDE +eq = Dxx(u(x, y)) + Dyy(u(x, y)) ~ -sin(pi * x) * sin(pi * y) + +# Boundary conditions +bcs = [u(0, y) ~ 0.0, u(1, y) ~ 0.0, + u(x, 0) ~ 0.0, u(x, 1) ~ 0.0] + +# Space and time domains +domains = [x ∈ Interval(0.0, 1.0), + y ∈ Interval(0.0, 1.0)] + +# Neural network +dim = 2 # number of dimensions +chain = Lux.Chain(Lux.Dense(dim, 9, Lux.σ), Lux.Dense(9, 9, Lux.σ), Lux.Dense(9, 1)) + +# Discretization +dx = 0.04 +discretization = BayesianPINN([chain], GridTraining(dx), dataset = [[dataset], nothing]) + +@named pde_system = PDESystem(eq, bcs, domains, [x, y], [u(x, y)]) + +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 5, + bcstd = [0.01, 0.01, 0.01, 0.01], + phystd = [0.005], + priorsNNw = (0.0, 2.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +xs = sol1.timepoints[1] +sol1.ensemblesol[1] +analytic_sol_func(x, y) = (sin(pi * x) * sin(pi * y)) / (2pi^2) + +dataset = hcat(u_real, xs') +u_predict = pmean(sol1.ensemblesol[1]) +u_real = [analytic_sol_func(xs[:, i][1], xs[:, i][2]) for i in 1:length(xs[1, :])] +@test u_predict≈u_real atol=0.8 \ No newline at end of file From 3e96e3db065e0d618996efc1c4e6318febdf78da Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Fri, 29 Mar 2024 22:25:27 +0530 Subject: [PATCH 39/53] update BPINN_PDEinvsol_tests.jl --- test/BPINN_PDEinvsol_tests.jl | 1113 +-------------------------------- 1 file changed, 1 insertion(+), 1112 deletions(-) diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 1fadc8a0a5..748ff21686 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -242,1115 +242,4 @@ u_predict = pmean(sol2.ensemblesol[1]) @test u_predict≈u_real atol=1.5 @test mean(u_predict .- u_real) < 0.1 -@test sol1.estimated_de_params[1]≈param atol=param * 0.3 - - - -println("Example 3: Lotka Volterra with New parameter estimation") -@parameters t α β γ δ -@variables x(..) y(..) - -Dt = Differential(t) -eqs = [Dt(x(t))*α ~ x(t) - β * x(t) * y(t), Dt(y(t))*γ ~ δ * x(t) * y(t) - y(t)] -bcs = [x(0) ~ 1.0, y(0) ~ 1.0] -domains = [t ∈ Interval(0.0, 7.0)] - -# Define the parameters' values -# params = [α => 1.0, β => 0.5, γ => 0.5, δ => 1.0] -# p = [1.5, 1.0, 3.0, 1.0] - -chainl = [ - Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh),Lux.Dense(6, 1)), - Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh),Lux.Dense(6, 1)) -] - -initl, st = Lux.setup(Random.default_rng(), chainl[1]) -initl1, st1 = Lux.setup(Random.default_rng(), chainl[2]) - -using NeuralPDE, Lux, OrdinaryDiffEq, Distributions, Random - -function lotka_volterra(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end -# initial-value problem. -u0 = [1.0, 1.0] -# p = [2/3, 2/3, 1/3.0, 1/3.0] -p = [1.5, 1.0, 3.0, 1.0] -tspan = (0.0, 7.0) -prob = ODEProblem(lotka_volterra, u0, tspan, p) -dt = 0.01 -solution = solve(prob, Tsit5(); saveat = dt) - - -# function moving_average_smoothing(data::Vector{T}, window_size::Int) where {T} -# smoothed_data = similar(data, T, length(data)) - -# for i in 1:length(data) -# start_idx = max(1, i - window_size) -# end_idx = min(length(data), i + window_size) -# smoothed_data[i] = mean(data[start_idx:end_idx]) -# end - -# return smoothed_data' -# end - -# Extract solution -time = solution.t -u = hcat(solution.u...) -time1=solution.t -u_noisy = u .+ u .* (0.3 .* randn(size(u))) - -plot(time,u[1,:]) -plot!(time,u[2,:]) -scatter!(time1,u_noisy[1,:]) -scatter!(time1,u_noisy[2,:]) - -# window_size = 5 -# smoothed_datasets = [moving_average_smoothing(u1[i, :], window_size) -# for i in 1:length(solution.u[1])] -# u2 = vcat(smoothed_datasets[1], smoothed_datasets[2]) -# Randomly select some points from the solution -num_points = 150 # Number of points to select -selected_indices = rand(1:size(u_noisy, 2), num_points) -upoints = [u_noisy[:, i] for i in selected_indices] -timepoints = [time[i] for i in selected_indices] -temp=hcat(upoints...) -dataset = [hcat(temp[i, :], timepoints) for i in 1:2] - -# plot(time,u[1,:]) -# plot!(time,u[2,:]) - -discretization = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, - dataset = [dataset, nothing]) - -@named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [x(t), y(t)], - [α, β, γ, δ], - defaults = Dict([α =>2, β => 3, γ =>3, δ =>2])) - -# creating dictionary for masking equations -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol3 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], progress = true) - -# time -# dataset -# chainl[1](time', sol3.estimated_nn_params[1], st)[1][1,:] -# plot!(time1, chainl[1](time1', sol3.estimated_nn_params[1], st)[1][1,:]) -# plot!(time1, chainl[2](time1', sol3.estimated_nn_params[2], st)[1][1,:]) -# plot!(time1, chainl[1](time1', sol5.estimated_nn_params[1], st)[1][1,:]) -# plot!(time1, chainl[2](time1', sol5.estimated_nn_params[2], st)[1][1,:]) -# time1 = collect(0.0:(1 / 100.0):8.0) - -sol4 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], progress = true -) - - -sol5_00 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.15, 0.15], - phystd = [0.15, 0.15], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5_0 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.05, 0.05], - phystd = [0.05, 0.05], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.1, 0.1], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -# 100 points(sol5_2 vs sol3) -sol5_2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.1, 0.1], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -# 100 points(sol5_2 vs sol3) -sol5_2_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.08, 0.08], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -# 100 points(sol5_2 vs sol3) -sol5_2_2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -# 50 datapoint 0-5 sol5 vs sol4 -# julia> sol4.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.549 ± 0.0058 -# 0.71 ± 0.0042 -# 0.408 ± 0.0063 -# 0.355 ± 0.0015 - -# julia> sol5.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.604 ± 0.0052 -# 0.702 ± 0.0034 -# 0.346 ± 0.0037 -# 0.335 ± 0.0013 - -# 100 datapoint 0-5 sol5_2 vs sol3 -# julia> sol3.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.598 ± 0.0037 -# 0.711 ± 0.0027 -# 0.399 ± 0.0032 -# 0.333 ± 0.0011 - -# julia> sol5_2.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.604 ± 0.0035 -# 0.686 ± 0.0026 -# 0.395 ± 0.0029 -# 0.328 ± 0.00095 - -# timespan for full dataset (0-8) -sol6 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], progress = true) - -sol5_3 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5_4 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5_5 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], progress = true -) - -sol7 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol5_5_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], progress = true -) - -sol7_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol7_2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.1, 0.1], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol7_3 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol7_4 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -lpfun = function f(chain::Chains) # function to compute the logpdf values - niter, nparams, nchains = size(chain) - lp = zeros(niter + nchains) # resulting logpdf values - for i = 1:nparams - lp += logpdf(MvNormal(Array(chain[:,i,:])) , dataset[1][:,1]') - lp += logpdf(MvNormal(Array(chain[:,i,:])) , dataset[1][:,2]') - end - return lp -end - -DIC, pD = dic(sol3.original.mcmc_chain, lpfun) -DIC1, pD1 = dic(sol4.original.mcmc_chain, lpfun) - -size(sol3.original.mcmc_chain) -Array(sol3.original.mcmc_chain[1,:,:]) -length(sol3.estimated_nn_params[1]) -chainl[1](time', sol3.estimated_nn_params[1], st)[1] - -data = [hcat(calculate_derivatives2(dataset[i][:, 2], dataset[1][:, 1]),dataset[i][:, 2]) for i in eachindex(dataset)] -dataset[1][:,1] -dataset[2] -plot!(dataset[1][:,2],dataset[1][:,1]) -eqs -sol5 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 200, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.02, 0.02], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(3, 2), - Normal(3, 2) - # LogNormal(1, 2), - # LogNormal(1, 2), - # LogNormal(1, 2), - # LogNormal(1, 2) - ], progress = true) - -# plot(time, chainl[1](time', sol2.estimated_nn_params[1], st)[1]) -# plot!(time, chainl[2](time', sol2.estimated_nn_params[2], st)[1]) - -sol6 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 200, - bcstd = [0.5, 0.5], - phystd = [0.5, 0.5], l2std = [0.02, 0.02], - priorsNNw = (0.0, 5.0), phystdnew = [0.5, 0.5], - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8),aa - param = [ - # LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3) - # Normal(3, 2), - # Normal(4, 2), - Normal(3, 2), - Normal(3, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -function calculate_derivatives2(indvar,depvar) - x̂, time = indvar,depvar - num_points = length(x̂) - # Initialize an array to store the derivative values. - derivatives = similar(x̂) - - for i in 2:(num_points - 1) - # Calculate the first-order derivative using central differences. - Δt_forward = time[i + 1] - time[i] - Δt_backward = time[i] - time[i - 1] - - derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) - - derivatives[i] = derivative - end - - # Derivatives at the endpoints can be calculated using forward or backward differences. - derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) - derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) - return derivatives -end -dataset[1] -dataset[2] -dataset[1][:,1]=calculate_derivatives2(dataset[1][:,2], dataset[1][:,1]) -dataset[2][:,1]=calculate_derivatives2(dataset[2][:,2], dataset[2][:,1]) -dataset[1] -dataset[2] -sol7 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 200, - bcstd = [0.5, 0.5], - phystd = [0.5, 0.5], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(0, 2), - Normal(0, 2) - # LogNormal(1, 2), - # LogNormal(1, 2), - # LogNormal(1, 2), - # LogNormal(1, 2) - ], progress = true) - -# plot(time, chainl[1](time', sol2.estimated_nn_params[1], st)[1]) -# plot!(time, chainl[2](time', sol2.estimated_nn_params[2], st)[1]) - -sol8 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), phystdnew = [0.1, 0.1], - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8),aa - param = [ - # LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3) - # Normal(3, 2), - # Normal(4, 2), - Normal(0, 2), - Normal(0, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -timepoints = collect(0.0:(1 / 100.0):9.0) -plot!(timepoints', chainl[1](timepoints', sol5_4.estimated_nn_params[1], st)[1]) -plot!(timepoints, chainl[2](timepoints', sol5_4.estimated_nn_params[2], st)[1]) - -using Plots, StatsPlots -plotly() - -plot(time, u[1, :]) -plot!(time, u[2, :]) -scatter!(time, u_noisy[1, :]) -scatter!(time, u_noisy[2, :]) -scatter!(discretization.dataset[1][1][:,2], discretization.dataset[1][1][:,1]) -scatter!(discretization.dataset[1][2][:,2], discretization.dataset[1][2][:,1]) - -# plot28(sol4 seems better vs sol3 plots, params seems similar) -plot!(sol3.timepoints[1]', sol3.ensemblesol[1],legend=nothing) -plot!(sol3.timepoints[2]', sol3.ensemblesol[2]) -plot!(sol4.timepoints[1]', sol4.ensemblesol[1]) -plot!(sol4.timepoints[2]', sol4.ensemblesol[2]) - -plot!(sol4_2.timepoints[1]', sol4_2.ensemblesol[1],legend=nothing) -plot!(sol4_2.timepoints[2]', sol4_2.ensemblesol[2]) -plot!(sol5_2.timepoints[1]', sol5_2.ensemblesol[1],legend=nothing) -plot!(sol5_2.timepoints[2]', sol5_2.ensemblesol[2]) - -plot!(sol4_3.timepoints[1]', sol4_3.ensemblesol[1],legend=nothing) -plot!(sol4_3.timepoints[2]', sol4_3.ensemblesol[2]) -plot!(sol5_3.timepoints[1]', sol5_3.ensemblesol[1]) -plot!(sol5_3.timepoints[2]', sol5_3.ensemblesol[2]) -plot!(sol5_4.timepoints[1]', sol5_4.ensemblesol[1],legend=nothing) -plot!(sol5_4.timepoints[2]', sol5_4.ensemblesol[2]) - - -# plot 36 sol4 vs sol5(params sol4 better, but plots sol5 "looks" better),plot 44(sol5 better than sol6 overall) -plot!(sol5.timepoints[1]', sol5.ensemblesol[1],legend=nothing) -plot!(sol5.timepoints[2]', sol5.ensemblesol[2]) -plot!(sol6.timepoints[1]', sol6.ensemblesol[1]) -plot!(sol6.timepoints[2]', sol6.ensemblesol[2]) - -# plot52 sol7 vs sol5(sol5 overall better plots, params?) -plot!(sol7.timepoints[1]', sol7.ensemblesol[1]) -plot!(sol7.timepoints[2]', sol7.ensemblesol[2]) - -# sol8,sol8_2,sol9,sol9_2 bad -plot!(sol8.timepoints[1]', sol8.ensemblesol[1]) -plot!(sol8.timepoints[2]', sol8.ensemblesol[2]) -plot!(sol8_2.timepoints[1]', sol8_2.ensemblesol[1]) -plot!(sol8_2.timepoints[2]', sol8_2.ensemblesol[2]) - -plot!(sol9.timepoints[1]', sol9.ensemblesol[1]) -plot!(sol9.timepoints[2]', sol9.ensemblesol[2]) -plot!(sol9_2.timepoints[1]', sol9_2.ensemblesol[1]) -plot!(sol9_2.timepoints[2]', sol9_2.ensemblesol[2]) - - -plot!(sol5_5.timepoints[1]', sol5_5.ensemblesol[1]) -plot!(sol5_5.timepoints[2]', sol5_5.ensemblesol[2],legend=nothing) - -plot!(sol5_5_1.timepoints[1]', sol5_5_1.ensemblesol[1]) -plot!(sol5_5_1.timepoints[2]', sol5_5_1.ensemblesol[2],legend=nothing) -plot!(sol7_1.timepoints[1]', sol7_1.ensemblesol[1]) -plot!(sol7_1.timepoints[2]', sol7_1.ensemblesol[2]) - -plot!(sol7_4.timepoints[1]', sol7_4.ensemblesol[1]) -plot!(sol7_4.timepoints[2]', sol7_4.ensemblesol[2]) - -plot!(sol5_2_1.timepoints[1]', sol5_2_1.ensemblesol[1],legend=nothing) -plot!(sol5_2_1.timepoints[2]', sol5_2_1.ensemblesol[2]) -plot!(sol5_2_2.timepoints[1]', sol5_2_2.ensemblesol[1],legend=nothing) -plot!(sol5_2_2.timepoints[2]', sol5_2_2.ensemblesol[2]) - -plot!(sol5_0.timepoints[1]', sol5_0.ensemblesol[1]) -plot!(sol5_0.timepoints[2]', sol5_0.ensemblesol[2],legend=nothing) - -plot!(sol5_00.timepoints[1]', sol5_00.ensemblesol[1]) -plot!(sol5_00.timepoints[2]', sol5_00.ensemblesol[2],legend=nothing) - -# test with lower number of points -# test same calls 2 times or more -# consider full range dataset case -# combination of all above - -# run 1 100 iters -sol5.estimated_de_params -sol6.estimated_de_params - -# run 2 200 iters -sol5.estimated_de_params -sol6.estimated_de_params - -# run 2 200 iters -sol3.estimated_de_params -sol4.estimated_de_params - -# p = [2/3, 2/3, 1/3, 1/3] -sol3.estimated_de_params -sol4.estimated_de_params -dataset[1] -eqs -α, β, γ, δ = p -p -# 1.0 -# 0.6666666666666666 -# 1.0 -# 0.33333333333333333 - -1/a -1/c -eqs -using StatsPlots -plotly() -plot(sol3.original.mcmc_chain) -plot(sol4.original.mcmc_chain) - -# 4-element Vector{Particles{Float64, 34}}: -# 1.23 ± 0.022 -# 0.858 ± 0.011 -# 3.04 ± 0.079 -# 1.03 ± 0.024 -# 4-element Vector{Particles{Float64, 34}}: -# 1.2 ± 0.0069 -# 0.835 ± 0.006 -# 3.22 ± 0.01 -# 1.08 ± 0.0053 -# # plot(time', chainl[1](time', sol1.estimated_nn_params[1], st)[1]) -# # plot!(time, chainl[2](time', sol1.estimated_nn_params[2], st)[1]) - -# sol3 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 500, -# bcstd = [0.05, 0.05], -# phystd = [0.005, 0.005], l2std = [0.1, 0.1], -# phystdnew = [0.5, 0.5], -# # Kernel = AdvancedHMC.NUTS(0.8), -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 50.0], -# param = [ -# Normal(0.0, 2), -# Normal(0.0, 2), -# Normal(0.0, 2), -# Normal(0.0, 2) -# ], -# Dict_differentials = Dict_differentials, progress = true) - -# sol = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 500, -# bcstd = [0.05, 0.05], -# phystd = [0.005, 0.005], l2std = [0.1, 0.1], -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 50.0], -# # Kernel = AdvancedHMC.NUTS(0.8), -# param = [ -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2) -# ], progress = true) - -# plot!(sol.timepoints[1]', sol.ensemblesol[1]) -# plot!(sol.timepoints[2]', sol.ensemblesol[2]) - -# sol1 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 500, -# bcstd = [0.05, 0.05], -# phystd = [0.005, 0.005], l2std = [0.1, 0.1], -# phystdnew = [0.5, 0.5], -# # Kernel = AdvancedHMC.NUTS(0.8), -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 50.0], -# param = [ -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2) -# ], -# Dict_differentials = Dict_differentials, progress = true) - -# plot!(sol1.timepoints[1]', sol1.ensemblesol[1]) -# plot!(sol1.timepoints[2]', sol1.ensemblesol[2]) - -sol = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 500, - bcstd = [0.05, 0.05], - phystd = [0.005, 0.005], l2std = [0.1, 0.1], - priorsNNw = (0.0, 10.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2) - ]) - -# plot!(sol.timepoints[1]', sol.ensemblesol[1]) -# plot!(sol.timepoints[2]', sol.ensemblesol[2]) - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 500, - bcstd = [0.05, 0.05], - phystd = [0.005, 0.005], l2std = [0.1, 0.1], - phystdnew = [0.5, 0.5], - # Kernel = AdvancedHMC.NUTS(0.8), - priorsNNw = (0.0, 10.0), - saveats = [1 / 50.0], - param = [ - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2) - ], - Dict_differentials = Dict_differentials) - -param = 2 * π -ts = vec(sol1.timepoints[1]) -u_real = [analytic_sol_func1(0.0, t) for t in ts] -u_predict = pmean(sol1.ensemblesol[1]) - -@test u_predict≈u_real atol=1.5 -@test mean(u_predict .- u_real) < 0.1 -@test sol1.estimated_de_params[1]≈param atol=param * 0.3 - -# points1 = [] -# for eq_arg in eq_args -# a = [] -# # for each (depvar,[indvar1..]) if indvari==indvar (eq_arg) -# for i in eachindex(symbols_input) -# if symbols_input[i][2] == eq_arg -# # include domain points of that depvar -# # each loss equation take domain matrix [points..;points..] -# push!(a, train_sets[i][:, 2:end]') -# end -# end -# # vcat as new row for next equation -# push!(points1, vcat(a...)) -# end -# println(points1 == points) - -# using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC -# import ModelingToolkit: Interval, infimum, supremum, Distributions -# using Plots, MonteCarloMeasurements - -# @parameters x, t, α -# @variables u(..) -# Dt = Differential(t) -# Dx = Differential(x) -# Dx2 = Differential(x)^2 -# Dx3 = Differential(x)^3 -# Dx4 = Differential(x)^4 - -# # α = 1 -# β = 4 -# γ = 1 -# eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 - -# u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -# du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 - -# bcs = [u(x, 0) ~ u_analytic(x, 0), -# u(-10, t) ~ u_analytic(-10, t), -# u(10, t) ~ u_analytic(10, t), -# Dx(u(-10, t)) ~ du(-10, t), -# Dx(u(10, t)) ~ du(10, t)] - -# # Space and time domains -# domains = [x ∈ Interval(-10.0, 10.0), -# t ∈ Interval(0.0, 1.0)] - -# # Discretization -# dx = 0.4; -# dt = 0.2; - -# # Function to compute analytical solution at a specific point (x, t) -# function u_analytic_point(x, t) -# z = -x / 2 + t -# return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -# end - -# # Function to generate the dataset matrix -# function generate_dataset_matrix(domains, dx, dt) -# x_values = -10:dx:10 -# t_values = 0.0:dt:1.0 - -# dataset = [] - -# for t in t_values -# for x in x_values -# u_value = u_analytic_point(x, t) -# push!(dataset, [u_value, x, t]) -# end -# end - -# return vcat([data' for data in dataset]...) -# end - -# datasetpde = [generate_dataset_matrix(domains, dx, dt)] - -# # noise to dataset -# noisydataset = deepcopy(datasetpde) -# noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ -# randn(size(noisydataset[1][:, 1])) .* 5 / 100 .* -# noisydataset[1][:, 1] - -# # plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") -# # plot!(noisydataset[1][:, 2], noisydataset[1][:, 1]) - -# # Neural network -# chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), -# Lux.Dense(8, 8, Lux.tanh), -# Lux.Dense(8, 1)) - -# discretization = NeuralPDE.BayesianPINN([chain], -# GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) - -# @named pde_system = PDESystem(eq, -# bcs, -# domains, -# [x, t], -# [u(x, t)], -# [α], -# defaults = Dict([α => 0.5])) - -# sol1 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 100, -# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], -# phystd = [1.0], l2std = [0.05], param = [Distributions.LogNormal(0.5, 2)], -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 100.0, 1 / 100.0], progress = true) - -# eqs = pde_system.eqs -# Dict_differentials = Dict() -# exps = toexpr.(eqs) -# nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -# sol2 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 100, -# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], -# phystd = [1.0], phystdnew = [0.05], l2std = [0.05], -# param = [Distributions.LogNormal(0.5, 2)], -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 100.0, 1 / 100.0], Dict_differentials = Dict_differentials, -# progress = true) - -# phi = discretization.phi[1] -# xs, ts = [infimum(d.domain):dx:supremum(d.domain) -# for (d, dx) in zip(domains, [dx / 10, dt])] -# u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] -# for t in ts] -# u_real = [[u_analytic(x, t) for x in xs] for t in ts] -# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) -# for x in xs] -# for t in ts] - -# # p1 = plot(xs, u_predict, title = "predict") -# # p2 = plot(xs, u_real, title = "analytic") -# # p3 = plot(xs, diff_u, title = "error") -# # plot(p1, p2, p3) - -# phi = discretization.phi[1] -# xs, ts = [infimum(d.domain):dx:supremum(d.domain) -# for (d, dx) in zip(domains, [dx / 10, dt])] -# u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] -# for t in ts] -# u_real = [[u_analytic(x, t) for x in xs] for t in ts] -# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) -# for x in xs] -# for t in ts] - -# # p1 = plot(xs, u_predict, title = "predict") -# # p2 = plot(xs, u_real, title = "analytic") -# # p3 = plot(xs, diff_u, title = "error") -# # plot(p1, p2, p3) - -@parameters t, p -@variables u(..) - -Dt = Differential(t) -eqs = Dt(u(t)) - cos(p * t) ~ 0 -bcs = [u(0) ~ 0.0] -domains = [t ∈ Interval(0.0, 2.0)] - -chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) -initl, st = Lux.setup(Random.default_rng(), chainl) - -@named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [u(t)], - [p], - defaults = Dict([p => 4.0])) - -analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) -timepoints = collect(0.0:(1 / 100.0):2.0) -u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] -u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) -dataset = [hcat(u1, timepoints)] - -discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, - dataset = [dataset, nothing]) - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(4.0, 2)], progress = true) - -param = 2 * π -ts = vec(sol1.timepoints[1]) -u_real = [analytic_sol_func1(0.0, t) for t in ts] -u_predict = pmean(sol1.ensemblesol[1]) - -@test u_predict≈u_real atol=0.1 -@test mean(u_predict .- u_real) < 0.01 -@test sol1.estimated_de_params[1]≈param atol=0.1 -sol1.estimated_de_params[1] - -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.02], phystdnew = [0.02], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(4.0, 2)], - Dict_differentials = Dict_differentials, - progress = true) - -param = 2 * π -ts_2 = vec(sol2.timepoints[1]) -u_real_2 = [analytic_sol_func1(0.0, t) for t in ts] -u_predict_2 = pmean(sol2.ensemblesol[1]) - -@test u_predict_2≈u_real_2 atol=0.1 -@test mean(u_predict_2 .- u_real_2) < 0.01 -@test sol2.estimated_de_params[1]≈param atol=0.1 -sol2.estimated_de_params[1] - -plot(ts_2, u_predict_2) -plot!(ts_2, u_real_2) - -@parameters t, σ_ -@variables x(..), y(..), z(..) -Dt = Differential(t) -eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), - Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), - Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] - -bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] -domains = [t ∈ Interval(0.0, 1.0)] - -input_ = length(domains) -n = 7 -chain = [ - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)) -] - -#Generate Data -function lorenz!(du, u, p, t) - du[1] = 10.0 * (u[2] - u[1]) - du[2] = u[1] * (28.0 - u[3]) - u[2] - du[3] = u[1] * u[2] - (8 / 3) * u[3] -end - -u0 = [1.0; 0.0; 0.0] -tspan = (0.0, 1.0) -prob = ODEProblem(lorenz!, u0, tspan) -sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) -ts = sol.t -us = hcat(sol.u...) -us = us .+ ((0.05 .* randn(size(us))) .* us) -ts_ = hcat(sol(ts).t...)[1, :] -dataset = [hcat(us[i, :], ts_) for i in 1:3] - -discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, - dataset = [dataset, nothing]) - -@named pde_system = PDESystem(eqs, bcs, domains, - [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 100, - bcstd = [0.3, 0.3, 0.3], - phystd = [0.1, 0.1, 0.1], - l2std = [1, 1, 1], - priorsNNw = (0.0, 1.0), - saveats = [0.01], - param = [Normal(14.0, 2)], progress = true) - -idealp = 10.0 -p_ = sol1.estimated_de_params[1] -@test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] -# @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] - -@parameters x y -@variables u(..) -Dxx = Differential(x)^2 -Dyy = Differential(y)^2 - -# 2D PDE -eq = Dxx(u(x, y)) + Dyy(u(x, y)) ~ -sin(pi * x) * sin(pi * y) - -# Boundary conditions -bcs = [u(0, y) ~ 0.0, u(1, y) ~ 0.0, - u(x, 0) ~ 0.0, u(x, 1) ~ 0.0] - -# Space and time domains -domains = [x ∈ Interval(0.0, 1.0), - y ∈ Interval(0.0, 1.0)] - -# Neural network -dim = 2 # number of dimensions -chain = Lux.Chain(Lux.Dense(dim, 9, Lux.σ), Lux.Dense(9, 9, Lux.σ), Lux.Dense(9, 1)) - -# Discretization -dx = 0.04 -discretization = BayesianPINN([chain], GridTraining(dx), dataset = [[dataset], nothing]) - -@named pde_system = PDESystem(eq, bcs, domains, [x, y], [u(x, y)]) - -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 5, - bcstd = [0.01, 0.01, 0.01, 0.01], - phystd = [0.005], - priorsNNw = (0.0, 2.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -xs = sol1.timepoints[1] -sol1.ensemblesol[1] -analytic_sol_func(x, y) = (sin(pi * x) * sin(pi * y)) / (2pi^2) - -dataset = hcat(u_real, xs') -u_predict = pmean(sol1.ensemblesol[1]) -u_real = [analytic_sol_func(xs[:, i][1], xs[:, i][2]) for i in 1:length(xs[1, :])] -@test u_predict≈u_real atol=0.8 \ No newline at end of file +@test sol1.estimated_de_params[1]≈param atol=param * 0.3 \ No newline at end of file From efbccda4297d4c89d1515ea943aee4df0b665bb7 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Fri, 29 Mar 2024 23:47:36 +0530 Subject: [PATCH 40/53] update training_strategies.jl --- src/training_strategies.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/training_strategies.jl b/src/training_strategies.jl index e33f490fe7..d128d7bb5f 100644 --- a/src/training_strategies.jl +++ b/src/training_strategies.jl @@ -48,7 +48,7 @@ end # include dataset points in pde_residual loglikelihood (BayesianPINN) function merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, - strategy::GridTraining, + strategy, datafree_pde_loss_function, datafree_bc_loss_function; train_sets_pde = nothing, train_sets_bc = nothing) @unpack domains, eqs, bcs, dict_indvars, dict_depvars, flat_init_params = pinnrep From 39ed5f6d58b4cdba3807a773d5b73b1d7e4d605d Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sat, 30 Mar 2024 00:06:01 +0530 Subject: [PATCH 41/53] update BPINN_PDEinvsol_tests.jl --- src/training_strategies.jl | 2 +- test/BPINN_PDEinvsol_tests.jl | 73 ++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/training_strategies.jl b/src/training_strategies.jl index d128d7bb5f..e33f490fe7 100644 --- a/src/training_strategies.jl +++ b/src/training_strategies.jl @@ -48,7 +48,7 @@ end # include dataset points in pde_residual loglikelihood (BayesianPINN) function merge_strategy_with_loglikelihood_function(pinnrep::PINNRepresentation, - strategy, + strategy::GridTraining, datafree_pde_loss_function, datafree_bc_loss_function; train_sets_pde = nothing, train_sets_bc = nothing) @unpack domains, eqs, bcs, dict_indvars, dict_depvars, flat_init_params = pinnrep diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 748ff21686..5cc8fe95fe 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -34,42 +34,43 @@ Random.seed!(100) u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) dataset = [hcat(u1, timepoints)] - # checking all training strategies - discretization = BayesianPINN([chainl], StochasticTraining(200), param_estim = true, - dataset = [dataset, nothing]) - - ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)]) - - discretization = BayesianPINN([chainl], QuasiRandomTraining(200), param_estim = true, - dataset = [dataset, nothing]) - - ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)]) - - discretization = BayesianPINN([chainl], QuadratureTraining(), param_estim = true, - dataset = [dataset, nothing]) - - ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)]) + # TODO: correct implementations + # # checking all training strategies + # discretization = BayesianPINN([chainl], StochasticTraining(200), param_estim = true, + # dataset = [dataset, nothing]) + + # ahmc_bayesian_pinn_pde(pde_system, + # discretization; + # draw_samples = 1500, + # bcstd = [0.05], + # phystd = [0.01], l2std = [0.01], + # priorsNNw = (0.0, 1.0), + # saveats = [1 / 50.0], + # param = [LogNormal(6.0, 0.5)]) + + # discretization = BayesianPINN([chainl], QuasiRandomTraining(200), param_estim = true, + # dataset = [dataset, nothing]) + + # ahmc_bayesian_pinn_pde(pde_system, + # discretization; + # draw_samples = 1500, + # bcstd = [0.05], + # phystd = [0.01], l2std = [0.01], + # priorsNNw = (0.0, 1.0), + # saveats = [1 / 50.0], + # param = [LogNormal(6.0, 0.5)]) + + # discretization = BayesianPINN([chainl], QuadratureTraining(), param_estim = true, + # dataset = [dataset, nothing]) + + # ahmc_bayesian_pinn_pde(pde_system, + # discretization; + # draw_samples = 1500, + # bcstd = [0.05], + # phystd = [0.01], l2std = [0.01], + # priorsNNw = (0.0, 1.0), + # saveats = [1 / 50.0], + # param = [LogNormal(6.0, 0.5)]) discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, dataset = [dataset, nothing]) From 27310635d9510c99e14ca61b6a278e2640c7a65b Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sun, 5 May 2024 01:50:15 +0530 Subject: [PATCH 42/53] changes from reviews --- src/collocated_estim.jl | 123 +--------------------------------------- 1 file changed, 2 insertions(+), 121 deletions(-) diff --git a/src/collocated_estim.jl b/src/collocated_estim.jl index 3902f74a27..0fe608e951 100644 --- a/src/collocated_estim.jl +++ b/src/collocated_estim.jl @@ -1,56 +1,14 @@ -# suggested extra loss function +# suggested extra loss function for ODE solver case function L2loss2(Tar::LogTargetDensity, θ) f = Tar.prob.f # parameter estimation chosen or not if Tar.extraparams > 0 - # deri_sol = deri_sol' autodiff = Tar.autodiff - # # Timepoints to enforce Physics - # dataset = Array(reduce(hcat, dataset)') - # t = dataset[end, :] - # û = dataset[1:(end - 1), :] - - # ode_params = Tar.extraparams == 1 ? - # θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : - # θ[((length(θ) - Tar.extraparams) + 1):length(θ)] - - # if length(û[:, 1]) == 1 - # physsol = [f(û[:, i][1], - # ode_params, - # t[i]) - # for i in 1:length(û[1, :])] - # else - # physsol = [f(û[:, i], - # ode_params, - # t[i]) - # for i in 1:length(û[1, :])] - # end - # #form of NN output matrix output dim x n - # deri_physsol = reduce(hcat, physsol) - - # > for perfect deriv(basically gradient matching in case of an ODEFunction) - # in case of PDE or general ODE we would want to reduce residue of f(du,u,p,t) - # if length(û[:, 1]) == 1 - # deri_sol = [f(û[:, i][1], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[1, :])] - # else - # deri_sol = [f(û[:, i], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[1, :])] - # end - # deri_sol = reduce(hcat, deri_sol) - # deri_sol = reduce(hcat, derivatives) - # Timepoints to enforce Physics t = Tar.dataset[end] u1 = Tar.dataset[2] û = Tar.dataset[1] - # Tar(t, θ[1:(length(θ) - Tar.extraparams)])' - # nnsol = NNodederi(Tar, t, θ[1:(length(θ) - Tar.extraparams)], autodiff) @@ -71,24 +29,7 @@ function L2loss2(Tar::LogTargetDensity, θ) end #form of NN output matrix output dim x n deri_physsol = reduce(hcat, physsol) - - # if length(Tar.prob.u0) == 1 - # nnsol = [f(û[i], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[:, 1])] - # else - # nnsol = [f([û[i], u1[i]], - # Tar.prob.p, - # t[i]) - # for i in 1:length(û[:, 1])] - # end - # form of NN output matrix output dim x n - # nnsol = reduce(hcat, nnsol) - - # > Instead of dataset gradients trying NN derivatives with dataset collocation - # # convert to matrix as nnsol - + physlogprob = 0 for i in 1:length(Tar.prob.u0) # can add phystd[i] for u[i] @@ -102,64 +43,4 @@ function L2loss2(Tar::LogTargetDensity, θ) else return 0 end -end - -# PDE(DU,U,P,T)=0 - -# Derivated via Central Diff -# function calculate_derivatives2(dataset) -# x̂, time = dataset -# num_points = length(x̂) -# # Initialize an array to store the derivative values. -# derivatives = similar(x̂) - -# for i in 2:(num_points - 1) -# # Calculate the first-order derivative using central differences. -# Δt_forward = time[i + 1] - time[i] -# Δt_backward = time[i] - time[i - 1] - -# derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) - -# derivatives[i] = derivative -# end - -# # Derivatives at the endpoints can be calculated using forward or backward differences. -# derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) -# derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) -# return derivatives -# end - -function calderivatives(prob, dataset) - chainflux = Flux.Chain(Flux.Dense(1, 8, tanh), Flux.Dense(8, 8, tanh), - Flux.Dense(8, 2)) |> Flux.f64 - # chainflux = Flux.Chain(Flux.Dense(1, 7, tanh), Flux.Dense(7, 1)) |> Flux.f64 - function loss(x, y) - # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1]) + - # Flux.mse.(prob.u0[2] .+ (prob.tspan[2] .- x)' .* chainflux(x)[2, :], y[2])) - # sum(Flux.mse.(prob.u0[1] .+ (prob.tspan[2] .- x)' .* chainflux(x)[1, :], y[1])) - sum(Flux.mse.(chainflux(x), y)) - end - optimizer = Flux.Optimise.ADAM(0.01) - epochs = 3000 - for epoch in 1:epochs - Flux.train!(loss, - Flux.params(chainflux), - [(dataset[end]', dataset[1:(end - 1)])], - optimizer) - end - - # A1 = (prob.u0' .+ - # (prob.tspan[2] .- (dataset[end]' .+ sqrt(eps(eltype(Float64)))))' .* - # chainflux(dataset[end]' .+ sqrt(eps(eltype(Float64))))') - - # A2 = (prob.u0' .+ - # (prob.tspan[2] .- (dataset[end]'))' .* - # chainflux(dataset[end]')') - - A1 = chainflux(dataset[end]' .+ sqrt(eps(eltype(dataset[end][1])))) - A2 = chainflux(dataset[end]') - - gradients = (A2 .- A1) ./ sqrt(eps(eltype(dataset[end][1]))) - - return gradients end \ No newline at end of file From a90c7304f95ebe49665cce2b894c94fbf7097d17 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 9 May 2024 02:07:58 +0530 Subject: [PATCH 43/53] Testing code for BPINN PDEs --- test/BPINN_PDEinvsol_tests.jl | 2397 ++++++++++++++++++++++++++++++++- 1 file changed, 2396 insertions(+), 1 deletion(-) diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 5cc8fe95fe..e756a3861f 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -243,4 +243,2399 @@ u_predict = pmean(sol2.ensemblesol[1]) @test u_predict≈u_real atol=1.5 @test mean(u_predict .- u_real) < 0.1 -@test sol1.estimated_de_params[1]≈param atol=param * 0.3 \ No newline at end of file +@test sol1.estimated_de_params[1]≈param atol=param * 0.3 + + +println("Example 3: Lotka Volterra with New parameter estimation") +@parameters t α β γ δ +@variables x(..) y(..) + +Dt = Differential(t) +eqs = [Dt(x(t)) * α ~ x(t) - β * x(t) * y(t), Dt(y(t)) * δ ~ x(t) * y(t) - y(t)*γ ] +bcs = [x(0) ~ 1.0, y(0) ~ 1.0] +domains = [t ∈ Interval(0.0, 7.0)] + +# Define the parameters' values +# α, β, γ, δ = p + +# regular equations +# dx = (1.5 - y) * x # prey +# dy = (x - 3.0) * y # predator +# p = [1.5, 1.0, 3.0, 1.0] non transformed values + +# transformed equations +# dx*0.666 = (1 - 0.666 * y) * x # prey +# dy*1.0 = (x - 3.0) * y # predator +# p = [0.666, 0.666, 3.0, 1.0] transformed values (change is scale also ensured!) + +chainl = [ + Lux.Chain(Lux.Dense(1, 5, sin), Lux.Dense(5, 5, sin),Lux.Dense(5, 1)), + Lux.Chain(Lux.Dense(1, 5, sin), Lux.Dense(5, 5, sin),Lux.Dense(5, 1)) +] + +initl, st = Lux.setup(Random.default_rng(), chainl[1]) +initl1, st1 = Lux.setup(Random.default_rng(), chainl[2]) + +using NeuralPDE, Lux, OrdinaryDiffEq, Distributions, Random + +function lotka_volterra(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end +# initial-value problem. +u0 = [1.0, 1.0] +# p = [2/3, 2/3, 1/3.0, 1/3.0] +p = [1.5, 1.0, 3.0, 1.0] +tspan = (0.0, 7.0) +prob = ODEProblem(lotka_volterra, u0, tspan, p) +dt = 0.01 +solution = solve(prob, Tsit5(); saveat = dt) +solution1 = solve(prob, Tsit5(); saveat = 0.02) + +function calculate_errors(approx_sol, solution_points) + # Check vector lengths match + if length(approx_sol) != length(solution_points) + error("Vectors must have the same length") + end + + # Calculate errors + n = length(approx_sol) + errors = randn(n) + for i in 1:n + errors[i] = solution_points[i] - approx_sol[i] + end + + # Calculate RMSE + rmse = sqrt(mean(errors.^2)) + + # Calculate MAE + mae = mean(abs.(errors)) + + # Calculate maximum absolute error + max_error = maximum(abs.(errors)) + + # Return dictionary with errors + return Dict( + "RMSE" => rmse, + "MAE" => mae, + "Max Abs Error" => max_error, + ) +end +u = hcat(solution1.u...) +u[1,:] +sol6_2.ensemblesol[1] + +a1=calculate_errors(pmean(sol6_1.ensemblesol[1]), u1[1,:]) +b1=calculate_errors(pmean(sol6_1.ensemblesol[2]), u1[2,:]) + +a=calculate_errors(pmean(sol6_2.ensemblesol[1]), u[1,:]) +b=calculate_errors(pmean(sol6_2.ensemblesol[2]), u[2,:]) + +c=calculate_errors(pmean(sol6_L2_2.ensemblesol[1]), u[1,:]) +d=calculate_errors(pmean(sol6_L2_2.ensemblesol[2]), u[2,:]) + +e=calculate_errors(pmean(sol6_L2_1.ensemblesol[1]), u[1,:]) +f=calculate_errors(pmean(sol6_L2_1.ensemblesol[2]), u[2,:]) + +g=calculate_errors(pmean(sol6_L2.ensemblesol[1]), u[1,:]) +h=calculate_errors(pmean(sol6_L2.ensemblesol[2]), u[2,:]) +sol6_2.ensemblesol[1] +sol6_2.ensemblesol[2] + +sol6_L2.ensemblesol[1] +sol6_L2.ensemblesol[2] + +# function moving_average_smoothing(data::Vector{T}, window_size::Int) where {T} +# smoothed_data = similar(data, T, length(data)) + +# for i in 1:length(data) +# start_idx = max(1, i - window_size) +# end_idx = min(length(data), i + window_size) +# smoothed_data[i] = mean(data[start_idx:end_idx]) +# end + +# return smoothed_data' +# end + +# Extract solution +time = solution.t +u = hcat(solution.u...) +time1=solution.t +u_noisy = u .+ u .* (0.2 .* randn(size(u))) +u_noisy0 = u .+ (3.0 .* rand(size(u)[1],size(u)[2]) .- 1.5) +u_noisy1 = u .+ (0.8.* randn(size(Array(solution)))) +u_noisy2 = u .+ (0.5.* randn(size(Array(solution)))) + +plot(time,u[1,:]) +plot!(time,u[2,:]) +scatter!(time1,u_noisy0[1,:]) +scatter!(time1,u_noisy0[2,:]) +scatter!(discretization_08_gaussian.dataset[1][1][:,2], discretization_08_gaussian.dataset[1][1][:,1]) +scatter!(discretization_08_gaussian.dataset[1][2][:,2], discretization_08_gaussian.dataset[1][2][:,1]) + +scatter!(discretization_05_gaussian.dataset[1][1][:,2], discretization_05_gaussian.dataset[1][1][:,1]) +scatter!(discretization_05_gaussian.dataset[1][2][:,2], discretization_05_gaussian.dataset[1][2][:,1]) +# discretization_05_gaussian.dataset[1][1][:,2] +# window_size = 5 +# smoothed_datasets = [moving_average_smoothing(u1[i, :], window_size) +# for i in 1:length(solution.u[1])] +# u2 = vcat(smoothed_datasets[1], smoothed_datasets[2]) +# Randomly select some points from the solution +num_points = 100 # Number of points to select +selected_indices = rand(1:size(u_noisy1, 2), num_points) +upoints = [u_noisy1[:, i] for i in selected_indices] +timepoints = [time[i] for i in selected_indices] +temp=hcat(upoints...) +dataset = [hcat(temp[i, :], timepoints) for i in 1:2] + +discretization_uniform = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, + dataset = [dataset, nothing]) +discretization_08_gaussian = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, + dataset = [dataset, nothing]) +discretization_05_gaussian = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, + dataset = [dataset, nothing]) + +discretization1 = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, + dataset = [dataset, nothing]) + +scatter!(discretization.dataset[1][1][:,2], discretization.dataset[1][1][:,1]) +scatter!(discretization.dataset[1][2][:,2], discretization.dataset[1][2][:,1]) + +sol = solve(prob, Tsit5(); saveat=0.1) +odedata = Array(sol) + 0.8 * randn(size(Array(sol))) + +@named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [x(t), y(t)], + [α, β, γ, δ], + defaults = Dict([α =>2, β => 2, γ =>2, δ =>2])) + +# creating dictionary for masking equations +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol3 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_uniform = ahmc_bayesian_pinn_pde(pde_system, + discretization_uniform; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_08_gaussian = ahmc_bayesian_pinn_pde(pde_system, + discretization_08_gaussian; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_05_gaussian = ahmc_bayesian_pinn_pde(pde_system, + discretization_05_gaussian; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + + +# more iterations for above +sol3_100_uniform_1000 = ahmc_bayesian_pinn_pde(pde_system, + discretization_uniform; + draw_samples = 1000, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_08_gaussian_1000 = ahmc_bayesian_pinn_pde(pde_system, + discretization_08_gaussian; + draw_samples = 1000, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_05_gaussian_1000 = ahmc_bayesian_pinn_pde(pde_system, + discretization_05_gaussian; + draw_samples = 1000, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + + +# more iterations for above + strict BC +sol3_100_uniform_1000_bc = ahmc_bayesian_pinn_pde(pde_system, + discretization_uniform; + draw_samples = 1000, + bcstd = [0.1, 0.1], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_08_gaussian_1000_bc = ahmc_bayesian_pinn_pde(pde_system, + discretization_08_gaussian; + draw_samples = 1000, + bcstd = [0.1, 0.1], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_08_gaussian_1000_bc_hard = ahmc_bayesian_pinn_pde(pde_system, + discretization_08_gaussian; + draw_samples = 1000, + bcstd = [0.05, 0.05], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_05_gaussian_1000_bc = ahmc_bayesian_pinn_pde(pde_system, + discretization_05_gaussian; + draw_samples = 1000, + bcstd = [0.1, 0.1], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_08_gaussian_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_08_gaussian; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol3_100_05_gaussian_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_05_gaussian; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +# time +# dataset +# chainl[1](time', sol3.estimated_nn_params[1], st)[1][1,:] +# plot!(time1, chainl[1](time1', sol3.estimated_nn_params[1], st)[1][1,:]) +# plot!(time1, chainl[2](time1', sol3.estimated_nn_params[2], st)[1][1,:]) +# plot!(time1, chainl[1](time1', sol5.estimated_nn_params[1], st)[1][1,:]) +# plot!(time1, chainl[2](time1', sol5.estimated_nn_params[2], st)[1][1,:]) +# time1 = collect(0.0:(1 / 100.0):8.0) + +sol4 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true +) + +sol4_0 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true +) + + +sol5_00 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5_0 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + + +# 70 points in dataset +sol6 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +# SOL6_1 VS SOL6_L2 +sol6_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol6_2 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol6_2_L2 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol6_3 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol6_4 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol6_L2 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol6_L2_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol6_L2_2 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.05, 0.05], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +# 50 datapoint 0-5 sol5 vs sol4 +# julia> sol4.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.549 ± 0.0058 +# 0.71 ± 0.0042 +# 0.408 ± 0.0063 +# 0.355 ± 0.0015 + +# julia> sol5.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.604 ± 0.0052 +# 0.702 ± 0.0034 +# 0.346 ± 0.0037 +# 0.335 ± 0.0013 + +# 100 datapoint 0-5 sol5_2 vs sol3 +# julia> sol3.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.598 ± 0.0037 +# 0.711 ± 0.0027 +# 0.399 ± 0.0032 +# 0.333 ± 0.0011 + +# julia> sol5_2.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.604 ± 0.0035 +# 0.686 ± 0.0026 +# 0.395 ± 0.0029 +# 0.328 ± 0.00095 + +# timespan for full dataset (0-8) +sol6 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], progress = true) + +sol5_3 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5_4 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5_5 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], progress = true +) + +sol7 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol5_5_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], progress = true +) + +sol7_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol7_2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.1, 0.1], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol7_3 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol7_4 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +lpfun = function f(chain::Chains) # function to compute the logpdf values + niter, nparams, nchains = size(chain) + lp = zeros(niter + nchains) # resulting logpdf values + for i = 1:nparams + lp += logpdf(MvNormal(Array(chain[:,i,:])) , dataset[1][:,1]') + lp += logpdf(MvNormal(Array(chain[:,i,:])) , dataset[1][:,2]') + end + return lp +end + +DIC, pD = dic(sol3.original.mcmc_chain, lpfun) +DIC1, pD1 = dic(sol4.original.mcmc_chain, lpfun) + +size(sol3.original.mcmc_chain) +Array(sol3.original.mcmc_chain[1,:,:]) +length(sol3.estimated_nn_params[1]) +chainl[1](time', sol3.estimated_nn_params[1], st)[1] + +data = [hcat(calculate_derivatives2(dataset[i][:, 2], dataset[1][:, 1]),dataset[i][:, 2]) for i in eachindex(dataset)] +dataset[1][:,1] +dataset[2] +plot!(dataset[1][:,2],dataset[1][:,1]) +eqs +sol5 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 200, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.02, 0.02], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(3, 2), + Normal(3, 2) + # LogNormal(1, 2), + # LogNormal(1, 2), + # LogNormal(1, 2), + # LogNormal(1, 2) + ], progress = true) + +# plot(time, chainl[1](time', sol2.estimated_nn_params[1], st)[1]) +# plot!(time, chainl[2](time', sol2.estimated_nn_params[2], st)[1]) + +sol6 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 200, + bcstd = [0.5, 0.5], + phystd = [0.5, 0.5], l2std = [0.02, 0.02], + priorsNNw = (0.0, 5.0), phystdnew = [0.5, 0.5], + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8),aa + param = [ + # LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3) + # Normal(3, 2), + # Normal(4, 2), + Normal(3, 2), + Normal(3, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +function calculate_derivatives2(indvar,depvar) + x̂, time = indvar,depvar + num_points = length(x̂) + # Initialize an array to store the derivative values. + derivatives = similar(x̂) + + for i in 2:(num_points - 1) + # Calculate the first-order derivative using central differences. + Δt_forward = time[i + 1] - time[i] + Δt_backward = time[i] - time[i - 1] + + derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) + + derivatives[i] = derivative + end + + # Derivatives at the endpoints can be calculated using forward or backward differences. + derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) + derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) + return derivatives +end +dataset[1] +dataset[2] +dataset[1][:,1]=calculate_derivatives2(dataset[1][:,2], dataset[1][:,1]) +dataset[2][:,1]=calculate_derivatives2(dataset[2][:,2], dataset[2][:,1]) +dataset[1] +dataset[2] +sol7 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 200, + bcstd = [0.5, 0.5], + phystd = [0.5, 0.5], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(0, 2), + Normal(0, 2) + # LogNormal(1, 2), + # LogNormal(1, 2), + # LogNormal(1, 2), + # LogNormal(1, 2) + ], progress = true) + +# plot(time, chainl[1](time', sol2.estimated_nn_params[1], st)[1]) +# plot!(time, chainl[2](time', sol2.estimated_nn_params[2], st)[1]) + +sol8 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), phystdnew = [0.1, 0.1], + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8),aa + param = [ + # LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3) + # Normal(3, 2), + # Normal(4, 2), + Normal(0, 2), + Normal(0, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +timepoints = collect(0.0:(1 / 100.0):9.0) +plot!(timepoints', chainl[1](timepoints', sol5_4.estimated_nn_params[1], st)[1]) +plot!(timepoints, chainl[2](timepoints', sol5_4.estimated_nn_params[2], st)[1]) + +sol_L2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true +) + +sol_NEW = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) +sol_L2_70 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true +) + +sol_NEW_70 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +using Plots, StatsPlots +plotly() + +plot(time, u[1, :]) +plot!(time, u[2, :]) +scatter!(time, u_noisy[1, :]) +scatter!(time, u_noisy[2, :]) +scatter!(discretization.dataset[1][1][:,2], discretization.dataset[1][1][:,1]) +scatter!(discretization.dataset[1][2][:,2], discretization.dataset[1][2][:,1]) + +scatter!(discretization1.dataset[1][1][:,2], discretization1.dataset[1][1][:,1],legend=nothing) +scatter!(discretization1.dataset[1][2][:,2], discretization1.dataset[1][2][:,1]) + +# plot28(sol4 seems better vs sol3 plots, params seems similar) +plot!(sol3.timepoints[1]', sol3.ensemblesol[1]) +plot!(sol3.timepoints[2]', sol3.ensemblesol[2]) +plot!(sol3_0.timepoints[1]', sol3_0.ensemblesol[1]) +plot!(sol3_0.timepoints[2]', sol3_0.ensemblesol[2]) + +plot!(sol4.timepoints[1]', sol4.ensemblesol[1]) +plot!(sol4.timepoints[2]', sol4.ensemblesol[2]) +plot!(sol4_0.timepoints[1]', sol4_0.ensemblesol[1]) +plot!(sol4_0.timepoints[2]', sol4_0.ensemblesol[2]) + +plot!(sol4_2.timepoints[1]', sol4_2.ensemblesol[1],legend=nothing) +plot!(sol4_2.timepoints[2]', sol4_2.ensemblesol[2]) +plot!(sol5_2.timepoints[1]', sol5_2.ensemblesol[1],legend=nothing) +plot!(sol5_2.timepoints[2]', sol5_2.ensemblesol[2]) + +plot!(sol4_3.timepoints[1]', sol4_3.ensemblesol[1],legend=nothing) +plot!(sol4_3.timepoints[2]', sol4_3.ensemblesol[2]) +plot!(sol5_3.timepoints[1]', sol5_3.ensemblesol[1]) +plot!(sol5_3.timepoints[2]', sol5_3.ensemblesol[2]) +plot!(sol5_4.timepoints[1]', sol5_4.ensemblesol[1],legend=nothing) +plot!(sol5_4.timepoints[2]', sol5_4.ensemblesol[2]) + + +# plot 36 sol4 vs sol5(params sol4 better, but plots sol5 "looks" better),plot 44(sol5 better than sol6 overall) +plot!(sol5.timepoints[1]', sol5.ensemblesol[1],legend=nothing) +plot!(sol5.timepoints[2]', sol5.ensemblesol[2]) + +plot!(sol5_0.timepoints[1]', sol5_0.ensemblesol[1],legend=nothing) +plot!(sol5_0.timepoints[2]', sol5_0.ensemblesol[2]) + +plot!(sol5_00.timepoints[1]', sol5_00.ensemblesol[1],legend=nothing) +plot!(sol5_00.timepoints[2]', sol5_00.ensemblesol[2]) + +plot!(sol6.timepoints[1]', sol6.ensemblesol[1]) +plot!(sol6.timepoints[2]', sol6.ensemblesol[2]) +plot!(sol6_L2.timepoints[1]', sol6_L2.ensemblesol[1]) +plot!(sol6_L2.timepoints[2]', sol6_L2.ensemblesol[2]) + +plot!(sol6_L2_1.timepoints[1]', sol6_L2_1.ensemblesol[1]) +plot!(sol6_L2_1.timepoints[2]', sol6_L2_1.ensemblesol[2]) + +plot!(sol6_L2_2.timepoints[1]', sol6_L2_2.ensemblesol[1]) +plot!(sol6_L2_2.timepoints[2]', sol6_L2_2.ensemblesol[2]) + +plot!(sol6_1.timepoints[1]', sol6_1.ensemblesol[1]) +plot!(sol6_1.timepoints[2]', sol6_1.ensemblesol[2]) +plot!(sol6_2.timepoints[1]', sol6_2.ensemblesol[1]) +plot!(sol6_2.timepoints[2]', sol6_2.ensemblesol[2],legend=nothing) +plot!(sol6_2_L2.timepoints[1]', sol6_2_L2.ensemblesol[1]) +plot!(sol6_2_L2.timepoints[2]', sol6_2_L2.ensemblesol[2],legend=nothing) + +# plot52 sol7 vs sol5(sol5 overall better plots, params?) +plot!(sol7.timepoints[1]', sol7.ensemblesol[1]) +plot!(sol7.timepoints[2]', sol7.ensemblesol[2]) + +# sol8,sol8_2,sol9,sol9_2 bad +plot!(sol8.timepoints[1]', sol8.ensemblesol[1]) +plot!(sol8.timepoints[2]', sol8.ensemblesol[2]) +plot!(sol8_2.timepoints[1]', sol8_2.ensemblesol[1]) +plot!(sol8_2.timepoints[2]', sol8_2.ensemblesol[2]) + +plot!(sol9.timepoints[1]', sol9.ensemblesol[1]) +plot!(sol9.timepoints[2]', sol9.ensemblesol[2]) +plot!(sol9_2.timepoints[1]', sol9_2.ensemblesol[1]) +plot!(sol9_2.timepoints[2]', sol9_2.ensemblesol[2]) + + +plot!(sol5_5.timepoints[1]', sol5_5.ensemblesol[1]) +plot!(sol5_5.timepoints[2]', sol5_5.ensemblesol[2],legend=nothing) + +plot!(sol5_5_1.timepoints[1]', sol5_5_1.ensemblesol[1]) +plot!(sol5_5_1.timepoints[2]', sol5_5_1.ensemblesol[2],legend=nothing) +plot!(sol7_1.timepoints[1]', sol7_1.ensemblesol[1]) +plot!(sol7_1.timepoints[2]', sol7_1.ensemblesol[2]) + +plot!(sol7_4.timepoints[1]', sol7_4.ensemblesol[1]) +plot!(sol7_4.timepoints[2]', sol7_4.ensemblesol[2]) + +plot!(sol5_2_1.timepoints[1]', sol5_2_1.ensemblesol[1],legend=nothing) +plot!(sol5_2_1.timepoints[2]', sol5_2_1.ensemblesol[2]) +plot!(sol5_2_2.timepoints[1]', sol5_2_2.ensemblesol[1],legend=nothing) +plot!(sol5_2_2.timepoints[2]', sol5_2_2.ensemblesol[2]) + +plot!(sol5_0.timepoints[1]', sol5_0.ensemblesol[1]) +plot!(sol5_0.timepoints[2]', sol5_0.ensemblesol[2],legend=nothing) + +plot!(sol5_00.timepoints[1]', sol5_00.ensemblesol[1],legend=nothing) +plot!(sol5_00.timepoints[2]', sol5_00.ensemblesol[2]) + +plot!(sol3_0.timepoints[1]', sol3_0.ensemblesol[1]) +plot!(sol3_0.timepoints[2]', sol3_0.ensemblesol[2],legend=nothing) +plot!(sol4_0.timepoints[1]', sol4_0.ensemblesol[1]) +plot!(sol4_0.timepoints[2]', sol4_0.ensemblesol[2],legend=nothing) + +plot!(sol3_100_05_gaussian.timepoints[1]', sol3_100_05_gaussian.ensemblesol[1]) +plot!(sol3_100_05_gaussian.timepoints[2]', sol3_100_05_gaussian.ensemblesol[2],legend=nothing) + +plot!(sol3_100_05_gaussian_new.timepoints[1]', sol3_100_05_gaussian_new.ensemblesol[1]) +plot!(sol3_100_05_gaussian_new.timepoints[2]', sol3_100_05_gaussian_new.ensemblesol[2]) + +plot!(sol3_100_08_gaussian.timepoints[1]', sol3_100_08_gaussian.ensemblesol[1]) +plot!(sol3_100_08_gaussian.timepoints[2]', sol3_100_08_gaussian.ensemblesol[2]) + +plot!(sol3_100_08_gaussian_new.timepoints[1]', sol3_100_08_gaussian_new.ensemblesol[1]) +plot!(sol3_100_08_gaussian_new.timepoints[2]', sol3_100_08_gaussian_new.ensemblesol[2],legend=nothing) + +plot!(sol3_100_uniform.timepoints[1]', sol3_100_uniform.ensemblesol[1]) +plot!(sol3_100_uniform.timepoints[2]', sol3_100_uniform.ensemblesol[2]) + +plot!(sol3_100_08_gaussian_1000.timepoints[1]', sol3_100_08_gaussian_1000.ensemblesol[1]) +plot!(sol3_100_08_gaussian_1000.timepoints[2]', sol3_100_08_gaussian_1000.ensemblesol[2]) + +plot!(sol3_100_05_gaussian_1000.timepoints[1]', sol3_100_05_gaussian_1000.ensemblesol[1]) +plot!(sol3_100_05_gaussian_1000.timepoints[2]', sol3_100_05_gaussian_1000.ensemblesol[2]) + +plot!(sol3_100_uniform_1000.timepoints[1]', sol3_100_uniform_1000.ensemblesol[1]) +plot!(sol3_100_uniform_1000.timepoints[2]', sol3_100_uniform_1000.ensemblesol[2]) + +plot!(sol3_100_08_gaussian_1000_bc.timepoints[1]', sol3_100_08_gaussian_1000_bc.ensemblesol[1]) +plot!(sol3_100_08_gaussian_1000_bc.timepoints[2]', sol3_100_08_gaussian_1000_bc.ensemblesol[2]) + +# test with lower number of points +# test same calls 2 times or more +# consider full range dataset case +# combination of all above + +# run 1 100 iters +sol5.estimated_de_params +sol6.estimated_de_params + +# run 2 200 iters +sol5.estimated_de_params +sol6.estimated_de_params + +# run 2 200 iters +sol3.estimated_de_params +sol4.estimated_de_params + +# p = [2/3, 2/3, 1/3, 1/3] +sol3.estimated_de_params +sol4.estimated_de_params +dataset[1] +eqs +α, β, γ, δ = p +p +# 1.0 +# 0.6666666666666666 +# 1.0 +# 0.33333333333333333 + +1/a +1/c +eqs +using StatsPlots +plotly() +plot(sol3.original.mcmc_chain) +plot(sol5_00.original.mcmc_chain) + +# 4-element Vector{Particles{Float64, 34}}: +# 1.23 ± 0.022 +# 0.858 ± 0.011 +# 3.04 ± 0.079 +# 1.03 ± 0.024 +# 4-element Vector{Particles{Float64, 34}}: +# 1.2 ± 0.0069 +# 0.835 ± 0.006 +# 3.22 ± 0.01 +# 1.08 ± 0.0053 +# # plot(time', chainl[1](time', sol1.estimated_nn_params[1], st)[1]) +# # plot!(time, chainl[2](time', sol1.estimated_nn_params[2], st)[1]) + +# sol3 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 500, +# bcstd = [0.05, 0.05], +# phystd = [0.005, 0.005], l2std = [0.1, 0.1], +# phystdnew = [0.5, 0.5], +# # Kernel = AdvancedHMC.NUTS(0.8), +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 50.0], +# param = [ +# Normal(0.0, 2), +# Normal(0.0, 2), +# Normal(0.0, 2), +# Normal(0.0, 2) +# ], +# Dict_differentials = Dict_differentials, progress = true) + +# sol = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 500, +# bcstd = [0.05, 0.05], +# phystd = [0.005, 0.005], l2std = [0.1, 0.1], +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 50.0], +# # Kernel = AdvancedHMC.NUTS(0.8), +# param = [ +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2) +# ], progress = true) + +# plot!(sol.timepoints[1]', sol.ensemblesol[1]) +# plot!(sol.timepoints[2]', sol.ensemblesol[2]) + +# sol1 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 500, +# bcstd = [0.05, 0.05], +# phystd = [0.005, 0.005], l2std = [0.1, 0.1], +# phystdnew = [0.5, 0.5], +# # Kernel = AdvancedHMC.NUTS(0.8), +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 50.0], +# param = [ +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2), +# Normal(1.0, 2) +# ], +# Dict_differentials = Dict_differentials, progress = true) + +# plot!(sol1.timepoints[1]', sol1.ensemblesol[1]) +# plot!(sol1.timepoints[2]', sol1.ensemblesol[2]) + +sol = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 500, + bcstd = [0.05, 0.05], + phystd = [0.005, 0.005], l2std = [0.1, 0.1], + priorsNNw = (0.0, 10.0), + saveats = [1 / 50.0], + # Kernel = AdvancedHMC.NUTS(0.8), + param = [ + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2) + ]) + +# plot!(sol.timepoints[1]', sol.ensemblesol[1]) +# plot!(sol.timepoints[2]', sol.ensemblesol[2]) + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 500, + bcstd = [0.05, 0.05], + phystd = [0.005, 0.005], l2std = [0.1, 0.1], + phystdnew = [0.5, 0.5], + # Kernel = AdvancedHMC.NUTS(0.8), + priorsNNw = (0.0, 10.0), + saveats = [1 / 50.0], + param = [ + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2), + Normal(1.0, 2) + ], + Dict_differentials = Dict_differentials) + +param = 2 * π +ts = vec(sol1.timepoints[1]) +u_real = [analytic_sol_func1(0.0, t) for t in ts] +u_predict = pmean(sol1.ensemblesol[1]) + +@test u_predict≈u_real atol=1.5 +@test mean(u_predict .- u_real) < 0.1 +@test sol1.estimated_de_params[1]≈param atol=param * 0.3 + +# points1 = [] +# for eq_arg in eq_args +# a = [] +# # for each (depvar,[indvar1..]) if indvari==indvar (eq_arg) +# for i in eachindex(symbols_input) +# if symbols_input[i][2] == eq_arg +# # include domain points of that depvar +# # each loss equation take domain matrix [points..;points..] +# push!(a, train_sets[i][:, 2:end]') +# end +# end +# # vcat as new row for next equation +# push!(points1, vcat(a...)) +# end +# println(points1 == points) + +# using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC +# import ModelingToolkit: Interval, infimum, supremum, Distributions +# using Plots, MonteCarloMeasurements + +# @parameters x, t, α +# @variables u(..) +# Dt = Differential(t) +# Dx = Differential(x) +# Dx2 = Differential(x)^2 +# Dx3 = Differential(x)^3 +# Dx4 = Differential(x)^4 + +# # α = 1 +# β = 4 +# γ = 1 +# eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 + +# u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +# du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 + +# bcs = [u(x, 0) ~ u_analytic(x, 0), +# u(-10, t) ~ u_analytic(-10, t), +# u(10, t) ~ u_analytic(10, t), +# Dx(u(-10, t)) ~ du(-10, t), +# Dx(u(10, t)) ~ du(10, t)] + +# # Space and time domains +# domains = [x ∈ Interval(-10.0, 10.0), +# t ∈ Interval(0.0, 1.0)] + +# # Discretization +# dx = 0.4; +# dt = 0.2; + +# # Function to compute analytical solution at a specific point (x, t) +# function u_analytic_point(x, t) +# z = -x / 2 + t +# return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +# end + +# # Function to generate the dataset matrix +# function generate_dataset_matrix(domains, dx, dt) +# x_values = -10:dx:10 +# t_values = 0.0:dt:1.0 + +# dataset = [] + +# for t in t_values +# for x in x_values +# u_value = u_analytic_point(x, t) +# push!(dataset, [u_value, x, t]) +# end +# end + +# return vcat([data' for data in dataset]...) +# end + +# datasetpde = [generate_dataset_matrix(domains, dx, dt)] + +# # noise to dataset +# noisydataset = deepcopy(datasetpde) +# noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ +# randn(size(noisydataset[1][:, 1])) .* 5 / 100 .* +# noisydataset[1][:, 1] + +# # plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") +# # plot!(noisydataset[1][:, 2], noisydataset[1][:, 1]) + +# # Neural network +# chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), +# Lux.Dense(8, 8, Lux.tanh), +# Lux.Dense(8, 1)) + +# discretization = NeuralPDE.BayesianPINN([chain], +# GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) + +# @named pde_system = PDESystem(eq, +# bcs, +# domains, +# [x, t], +# [u(x, t)], +# [α], +# defaults = Dict([α => 0.5])) + +# sol1 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 100, +# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], +# phystd = [1.0], l2std = [0.05], param = [Distributions.LogNormal(0.5, 2)], +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 100.0, 1 / 100.0], progress = true) + +# eqs = pde_system.eqs +# Dict_differentials = Dict() +# exps = toexpr.(eqs) +# nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +# sol2 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 100, +# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], +# phystd = [1.0], phystdnew = [0.05], l2std = [0.05], +# param = [Distributions.LogNormal(0.5, 2)], +# priorsNNw = (0.0, 10.0), +# saveats = [1 / 100.0, 1 / 100.0], Dict_differentials = Dict_differentials, +# progress = true) + +# phi = discretization.phi[1] +# xs, ts = [infimum(d.domain):dx:supremum(d.domain) +# for (d, dx) in zip(domains, [dx / 10, dt])] +# u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] +# for t in ts] +# u_real = [[u_analytic(x, t) for x in xs] for t in ts] +# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) +# for x in xs] +# for t in ts] + +# # p1 = plot(xs, u_predict, title = "predict") +# # p2 = plot(xs, u_real, title = "analytic") +# # p3 = plot(xs, diff_u, title = "error") +# # plot(p1, p2, p3) + +# phi = discretization.phi[1] +# xs, ts = [infimum(d.domain):dx:supremum(d.domain) +# for (d, dx) in zip(domains, [dx / 10, dt])] +# u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] +# for t in ts] +# u_real = [[u_analytic(x, t) for x in xs] for t in ts] +# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) +# for x in xs] +# for t in ts] + +# # p1 = plot(xs, u_predict, title = "predict") +# # p2 = plot(xs, u_real, title = "analytic") +# # p3 = plot(xs, diff_u, title = "error") +# # plot(p1, p2, p3) + +@parameters t, p +@variables u(..) + +Dt = Differential(t) +eqs = Dt(u(t)) - cos(p * t) ~ 0 +bcs = [u(0) ~ 0.0] +domains = [t ∈ Interval(0.0, 2.0)] + +chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) +initl, st = Lux.setup(Random.default_rng(), chainl) + +@named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [u(t)], + [p], + defaults = Dict([p => 4.0])) + +analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) +timepoints = collect(0.0:(1 / 100.0):2.0) +u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] +u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) +dataset = [hcat(u1, timepoints)] + +discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, + dataset = [dataset, nothing]) + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(4.0, 2)], progress = true) + +param = 2 * π +ts = vec(sol1.timepoints[1]) +u_real = [analytic_sol_func1(0.0, t) for t in ts] +u_predict = pmean(sol1.ensemblesol[1]) + +@test u_predict≈u_real atol=0.1 +@test mean(u_predict .- u_real) < 0.01 +@test sol1.estimated_de_params[1]≈param atol=0.1 +sol1.estimated_de_params[1] + +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.02], phystdnew = [0.02], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(4.0, 2)], + Dict_differentials = Dict_differentials, + progress = true) + +param = 2 * π +ts_2 = vec(sol2.timepoints[1]) +u_real_2 = [analytic_sol_func1(0.0, t) for t in ts] +u_predict_2 = pmean(sol2.ensemblesol[1]) + +@test u_predict_2≈u_real_2 atol=0.1 +@test mean(u_predict_2 .- u_real_2) < 0.01 +@test sol2.estimated_de_params[1]≈param atol=0.1 +sol2.estimated_de_params[1] + +plot(ts_2, u_predict_2) +plot!(ts_2, u_real_2) + +@parameters t, σ_ +@variables x(..), y(..), z(..) +Dt = Differential(t) +eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), + Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), + Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] + +bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] +domains = [t ∈ Interval(0.0, 1.0)] + +input_ = length(domains) +n = 7 +chain = [ + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)) +] + +#Generate Data +function lorenz!(du, u, p, t) + du[1] = 10.0 * (u[2] - u[1]) + du[2] = u[1] * (28.0 - u[3]) - u[2] + du[3] = u[1] * u[2] - (8 / 3) * u[3] +end + +u0 = [1.0; 0.0; 0.0] +tspan = (0.0, 1.0) +prob = ODEProblem(lorenz!, u0, tspan) +sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) +ts = sol.t +us = hcat(sol.u...) +us = us .+ ((0.05 .* randn(size(us))) .* us) +ts_ = hcat(sol(ts).t...)[1, :] +dataset = [hcat(us[i, :], ts_) for i in 1:3] + +discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, + dataset = [dataset, nothing]) + +@named pde_system = PDESystem(eqs, bcs, domains, + [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 100, + bcstd = [0.3, 0.3, 0.3], + phystd = [0.1, 0.1, 0.1], + l2std = [1, 1, 1], + priorsNNw = (0.0, 1.0), + saveats = [0.01], + param = [Normal(14.0, 2)], progress = true) + +idealp = 10.0 +p_ = sol1.estimated_de_params[1] +@test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] +# @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] + +@parameters x y +@variables u(..) +Dxx = Differential(x)^2 +Dyy = Differential(y)^2 + +# 2D PDE +eq = Dxx(u(x, y)) + Dyy(u(x, y)) ~ -sin(pi * x) * sin(pi * y) + +# Boundary conditions +bcs = [u(0, y) ~ 0.0, u(1, y) ~ 0.0, + u(x, 0) ~ 0.0, u(x, 1) ~ 0.0] + +# Space and time domains +domains = [x ∈ Interval(0.0, 1.0), + y ∈ Interval(0.0, 1.0)] + +# Neural network +dim = 2 # number of dimensions +chain = Lux.Chain(Lux.Dense(dim, 9, Lux.σ), Lux.Dense(9, 9, Lux.σ), Lux.Dense(9, 1)) + +# Discretization +dx = 0.04 +discretization = BayesianPINN([chain], GridTraining(dx), dataset = [[dataset], nothing]) + +@named pde_system = PDESystem(eq, bcs, domains, [x, y], [u(x, y)]) + +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 5, + bcstd = [0.01, 0.01, 0.01, 0.01], + phystd = [0.005], + priorsNNw = (0.0, 2.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +xs = sol1.timepoints[1] +sol1.ensemblesol[1] +analytic_sol_func(x, y) = (sin(pi * x) * sin(pi * y)) / (2pi^2) + +dataset = hcat(u_real, xs') +u_predict = pmean(sol1.ensemblesol[1]) +u_real = [analytic_sol_func(xs[:, i][1], xs[:, i][2]) for i in 1:length(xs[1, :])] +@test u_predict≈u_real atol=0.8 + +using NeuralPDE, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC +import ModelingToolkit: Interval, infimum, supremum, Distributions +using Plots, MonteCarloMeasurements + +@parameters x t +@variables u(..) + +Dt = Differential(t) +Dx = Differential(x) +Dxx = Dx^2 +α = 0.05 +# Burger's equation +eq = Dt(u(t, x)) + u(t, x) * Dx(u(t, x)) - α * Dxx(u(t, x)) ~ 0 + +# boundary conditions +bcs = [ + u(0.0, x) ~ -sin(π * x), + u(t, -1.0) ~ 0.0, + u(t, 1.0) ~ 0.0 +] + +domains = [t ∈ Interval(0.0, 1.0), x ∈ Interval(-1.0, 1.0)] + +# Neural network +chain = Lux.Chain(Dense(2, 10, Lux.σ), Dense(10, 10, Lux.σ), Dense(10, 1)) +strategy = NeuralPDE.QuadratureTraining(; abstol = 1e-6, reltol = 1e-6, batch = 200) + +indvars = [t, x] +depvars = [u(t, x)] +@named pde_system = PDESystem(eq, bcs, domains, indvars, depvars) + +# KS EQUATION +using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC +import ModelingToolkit: Interval, infimum, supremum, Distributions +using Plots, MonteCarloMeasurements, StatsPlots + +@parameters x, t, α +@variables u(..) +Dt = Differential(t) +Dx = Differential(x) +Dx2 = Differential(x)^2 +Dx3 = Differential(x)^3 +Dx4 = Differential(x)^4 + +# α = 1 +β = 4 +γ = 1 +eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 + +u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 + +bcs = [u(x, 0) ~ u_analytic(x, 0), + u(-10, t) ~ u_analytic(-10, t), + u(10, t) ~ u_analytic(10, t), + Dx(u(-10, t)) ~ du(-10, t), + Dx(u(10, t)) ~ du(10, t)] + +# Space and time domains +domains = [x ∈ Interval(-10.0, 10.0), + t ∈ Interval(0.0, 1.0)] + +# Discretization +dx = 0.4; +dt = 0.2; + +# Function to compute analytical solution at a specific point (x, t) +function u_analytic_point(x, t) + z = -x / 2 + t + return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +end + +# Function to generate the dataset matrix +function generate_dataset_matrix(domains, dx, dt, xlim, tlim) + x_values = xlim[1]:dx:xlim[2] + t_values = tlim[1]:dt:tlim[2] + + dataset = [] + + for t in t_values + for x in x_values + u_value = u_analytic_point(x, t) + push!(dataset, [u_value, x, t]) + end + end + + return vcat([data' for data in dataset]...) +end + +# x_values = -10:dx:10 +# t_values = 0.0:dt:1.0 + +# dataset = [] + +# for t in t_values +# for x in x_values +# u_value = u_analytic_point(x, t) +# push!(dataset, [u_value, x, t]) +# end +# end +# dataset +# pop= vcat([data' for data in dataset]...) + +datasetpde = [generate_dataset_matrix(domains, dx, dt, [-10,10], [0.0,1.0])] + +datasetpde_new = [generate_dataset_matrix(domains, dx, dt, [-10,0], [0.0,1.0])] + +# noise to dataset +noisydataset = deepcopy(datasetpde) +noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ (randn(size(noisydataset[1][:, 1])) .* 0.8) + +noisydataset_new = deepcopy(datasetpde_new) +noisydataset_new[1][:, 1] = noisydataset_new[1][:, 1] .+ (randn(size(noisydataset_new[1][:, 1])) .* 0.8) + +# a=discretization_new.dataset[1] + +plotly() +plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") +# scatter!(a[1][:, 2], a[1][:, 1]) +scatter!(noisydataset[1][:, 2], noisydataset[1][:, 1]) + +plot(datasetpde[1][:, 2],datasetpde[1][:, 3], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") +scatter!(noisydataset[1][:, 2],noisydataset[1][:, 3], noisydataset[1][:, 1]) + +plotly() +plot(datasetpde_new[1][:, 2], datasetpde_new[1][:, 1], title = "Dataset from Analytical Solution") +scatter!(noisydataset_new[1][:, 2], noisydataset_new[1][:, 1]) + +plot(datasetpde_new[1][:, 2],datasetpde_new[1][:, 3], datasetpde_new[1][:, 1], title = "Dataset from Analytical Solution") +scatter!(noisydataset_new[1][:, 2],noisydataset_new[1][:, 3], noisydataset_new[1][:, 1]) + +noise_std = 1.4 +original_data = datasetpde[1][:, 1] +original_std = std(original_data) +ratio = noise_std / original_std + + +using StatsPlots +plot(sol1.original.mcmc_chain) +plot(sol2.original.mcmc_chain) + +plot(sol0_new.original.mcmc_chain) +plot(sol2_new.original.mcmc_chain) + +# Neural network +chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), + Lux.Dense(8, 8, Lux.tanh), + Lux.Dense(8, 1)) + +chain_more = Lux.Chain(Lux.Dense(2, 10, Lux.tanh), + Lux.Dense(10, 10, Lux.tanh), + Lux.Dense(10, 1)) +# chain = Lux.Chain(Lux.Dense(2, 8, Lux.σ), +# Lux.Dense(8, 8, Lux.σ), +# Lux.Dense(8, 1)) + +discretization = NeuralPDE.BayesianPINN([chain], + GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) + +discretization_more = NeuralPDE.BayesianPINN([chain_more], + GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) + +discretization_new = NeuralPDE.BayesianPINN([chain], + GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset_new, nothing]) + + +@named pde_system = PDESystem(eq, + bcs, + domains, + [x, t], + [u(x, t)], + [α], + defaults = Dict([α => 2.0])) + +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +Dict_differentials + +plot(sol1.original.mcmc_chain) +meanplot(sol1.original.mcmc_chain) +autocorplot(sol1.original.mcmc_chain) +traceplot(sol1.original.mcmc_chain) + +plot(sol2.original.mcmc_chain) +meanplot(sol2.original.mcmc_chain) +autocorplot(sol2.original.mcmc_chain) +traceplot(sol2.original.mcmc_chain) + +plot(sol0_new.original.mcmc_chain) +meanplot(sol0_new.original.mcmc_chain) +autocorplot(sol0_new.original.mcmc_chain) + +plot(sol2_new.original.mcmc_chain) +meanplot(sol2_new.original.mcmc_chain) +autocorplot(sol2_new.original.mcmc_chain) + +plot(sol3_new.original.mcmc_chain) +meanplot(sol3_new.original.mcmc_chain) +autocorplot(sol3_new.original.mcmc_chain) + + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.5], + phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol1_more = ahmc_bayesian_pinn_pde(pde_system, + discretization_more; + draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.5], + phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol1_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 90, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.7], + phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +sol2_more = ahmc_bayesian_pinn_pde(pde_system, + discretization_more; + draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +# julia> sol2 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 85, Kernel = AdvancedHMC.NUTS(0.8), +# bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], +# phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, +# priorsNNw = (0.0, 3.0), +# saveats = [1 / 100.0, 1 / 100.0], +# progress = true) +# ┌ Info: Current Physics Log-likelihood : +# └ ℓπ.full_loglikelihood(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -415167 +# ┌ Info: Current Prior Log-likelihood : +# └ priorlogpdf(ℓπ, initial_θ) = -214.1825373360679 +# ┌ Info: Current MSE against dataset Log-likelihood : +# └ L2LossData(ℓπ, initial_θ) = -109309.44973223892 +# Sampling 100%|███████████████████████████████| Time: 0:14:50 +# iterations: 85 +# ratio_divergent_transitions: 0.0 +# ratio_divergent_transitions_during_adaption: 0.02 +# n_steps: 127 +# is_accept: true +# acceptance_rate: 0.9982795867682919 +# log_density: -3832.934953640867 +# hamiltonian_energy: 4145.005901868316 +# hamiltonian_energy_error: -0.07863051782624098 +# max_hamiltonian_energy_error: -0.16790754244266282 +# tree_depth: 7 +# numerical_error: false +# step_size: 0.00018186972987192408 +# nom_step_size: 0.00018186972987192408 +# is_adapt: false +# mass_matrix: DiagEuclideanMetric([1.0, 1. +# [ Info: Sampling Complete. +# ┌ Info: Current Physics Log-likelihood : +# └ ℓπ.full_loglikelihood(setparameters(ℓπ, samples[end]), ℓπ.allstd) = -132 +# ┌ Info: Current Prior Log-likelihood : +# └ priorlogpdf(ℓπ, samples[end]) = -219.17544656823006 +# ┌ Info: Current MSE against dataset Log-likelihood : +# └ L2LossData(ℓπ, samples[end]) = -3481.509412470054 + +# julia> sol1 = ahmc_bayesian_pinn_pde(pde_system, +# discretization; +# draw_samples = 90, Kernel = AdvancedHMC.NUTS(0.8), +# bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.7], +# phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, +# priorsNNw = (0.0, 3.0), +# saveats = [1 / 100.0, 1 / 100.0], +# Dict_differentials = Dict_differentials, +# progress = true) +# ┌ Info: Current Physics Log-likelihood : +# └ ℓπ.full_loglikelihood(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -394622 +# ┌ Info: Current Prior Log-likelihood : +# └ priorlogpdf(ℓπ, initial_θ) = -214.1657203956881 +# ┌ Info: Current MSE against dataset Log-likelihood : +# └ L2LossData(ℓπ, initial_θ) = -107600.2750860966 +# ┌ Info: Current L2_LOSSY : +# └ ℓπ.L2_loss2(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -352.339686469935 +# Sampling 100%|███████████████████████████████| Time: 0:38:42 +# iterations: 90 +# ratio_divergent_transitions: 0.24 +# ratio_divergent_transitions_during_adaption: 0.02 +# n_steps: 34 +# is_accept: true +# acceptance_rate: 0.0755469536430885 +# log_density: -6535.135018473582 +# hamiltonian_energy: 6681.540376258076 +# hamiltonian_energy_error: -1.7097735125544204 +# max_hamiltonian_energy_error: 1216.239238705054 +# tree_depth: 5 +# numerical_error: true +# step_size: 0.0004111092751764056 +# nom_step_size: 0.0004111092751764056 +# is_adapt: false +# mass_matrix: DiagEuclideanMetric([1.0, 1. +# [ Info: Sampling Complete. +# ┌ Info: Current Physics Log-likelihood : +# └ ℓπ.full_loglikelihood(setparameters(ℓπ, samples[end]), ℓπ.allstd) = -272 +# ┌ Info: Current Prior Log-likelihood : +# └ priorlogpdf(ℓπ, samples[end]) = -218.6535874132563 +# ┌ Info: Current MSE against dataset Log-likelihood : +# └ L2LossData(ℓπ, samples[end]) = -3573.449092586736 +# ┌ Info: Current L2_LOSSY : +# └ ℓπ.L2_loss2(setparameters(ℓπ, samples[end]), ℓπ.allstd) = -2470.35523478 + +using MCMCChains +println(summarize(sol1.original.mcmc_chain)) +plot(sol1.original.mcmc_chain) + +sol3 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 100, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.7], l2std = [0.15], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 3.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +sol4 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1_1.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_1.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol3.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol3.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol4.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol4.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +sol0_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], + phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + + +julia> sol5_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 170, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +sol1_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], + phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol1_1_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.5], + phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol1_2_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 150, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.1], + phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 3.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol1_3_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 150, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], + phystd = [0.3], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 3.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol2_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 140, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + + +sol3_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 140, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +sol4_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 160, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +sol5_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 170, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +# phi = discretization.phi[1] + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol0_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol0_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1_1_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_1_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1_2_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_2_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1_3_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_3_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol2_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol3_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol3_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol4_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol4_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol5_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol5_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol5_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol5_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] +plotly() +p1 = plot(ts, xs, u_predict, title = "predict") +p2 = plot(ts, xs, u_real, title = "analytic") +p3 = plot(ts, xs, diff_u, title = "error") +plot(p1, p2, p3) +# julia> sol0_new = ahmc_bayesian_pinn_pde(pde_system, +# discretization_new; +# draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), +# bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], +# phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], +# priorsNNw = (0.0, 1.0), +# saveats = [1 / 100.0, 1 / 100.0], +# Dict_differentials = Dict_differentials, +# progress = true) +# ┌ Info: Current Physics Log-likelihood : +# └ ℓπ.full_loglikelihood(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -398314.38213382766 +# ┌ Info: Current Prior Log-likelihood : +# └ priorlogpdf(ℓπ, initial_θ) = -104.7365701596561 +# ┌ Info: Current MSE against dataset Log-likelihood : +# └ L2LossData(ℓπ, initial_θ) = -58553.36940699288 +# ┌ Info: Current L2_LOSSY : +# └ ℓπ.L2_loss2(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -281.85131447737575 +# Sampling 100%|███████████████████████████████| Time: 0:26:00 +# iterations: 110 +# ratio_divergent_transitions: 0.2 +# ratio_divergent_transitions_during_adaption: 0.03 +# n_steps: 11 +# is_accept: true +# acceptance_rate: 0.0024891070448310416 +# log_density: -13158.729119075539 +# hamiltonian_energy: 13212.763613683248 +# hamiltonian_energy_error: 0.0 +# max_hamiltonian_energy_error: 1492.7356803165876 +# tree_depth: 3 +# numerical_error: true +# step_size: 0.0002145156661425442 +# nom_step_size: 0.0002145156661425442 +# is_adapt: false +# mass_matrix: DiagEuclideanMetric([1.0, 1.0, 1.0, 1.0, 1.0, 1 ...]) +# [ Info: Sampling Complete. +# ┌ Info: Current Physics Log-likelihood : +# └ ℓπ.full_loglikelihood(setparameters(ℓπ, samples[end]), ℓπ.allstd) = -908.7769621441158 +# ┌ Info: Current Prior Log-likelihood : +# └ priorlogpdf(ℓπ, samples[end]) = -136.87645881663929 +# ┌ Info: Current MSE against dataset Log-likelihood : +# └ L2LossData(ℓπ, samples[end]) = -1404.7102059521355 +# ┌ Info: Current L2_LOSSY : +# └ ℓπ.L2_loss2(setparameters(ℓπ, samples[end]), ℓπ.allstd) = -10708.363203924739 + +# julia> sol2_new = ahmc_bayesian_pinn_pde(pde_system, +# discretization_new; +# draw_samples = 140, Kernel = AdvancedHMC.NUTS(0.8), +# bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], +# phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], +# priorsNNw = (0.0, 1.0), +# saveats = [1 / 100.0, 1 / 100.0], +# progress = true) +# ┌ Info: Current Physics Log-likelihood : +# └ ℓπ.full_loglikelihood(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -397526.19267355377 +# ┌ Info: Current Prior Log-likelihood : +# └ priorlogpdf(ℓπ, initial_θ) = -105.03439044100367 +# ┌ Info: Current MSE against dataset Log-likelihood : +# └ L2LossData(ℓπ, initial_θ) = -60957.24454333089 +# Sampling 99%|███████████████████████████████| ETA: 0:00:10 +# iterations: 140 +# ratio_divergent_transitions: 0.0 +# ratio_divergent_transitions_during_adaption: 0.01 +# n_steps: 1023 +# is_accept: true +# acceptance_rate: 0.972620625460237 +# log_density: -1513.1769839294327 +# hamiltonian_energy: 8709.204139640105 +# hamiltonian_energy_error: -0.4925547801958601 +# max_hamiltonian_energy_error: -1.7861646674082294 +# tree_depth: 10 +# numerical_error: false +# step_size: 0.00011428277138492957 +# nom_step_size: 0.00011428277138492957 +# is_adapt: false +# mass_matrix: DiagEuclideanMetric([1.0, 1.0, 1.0, 1.0, 1.0, 1 ...]) +# [ Info: Sampling Complete. +# ┌ Info: Current Physics Log-likelihood : +# └ ℓπ.full_loglikelihood(setparameters(ℓπ, samples[end]), ℓπ.allstd) = 115.103823132341 +# ┌ Info: Current Prior Log-likelihood : +# └ priorlogpdf(ℓπ, samples[end]) = -198.39103020815858 +# ┌ Info: Current MSE against dataset Log-likelihood : +# └ L2LossData(ℓπ, samples[end]) = -1429.7843027541815 From 2331614f0ee40d89e2adcb86edca8fa18a73ad6f Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Thu, 9 May 2024 03:21:00 +0530 Subject: [PATCH 44/53] spelling corrections, cleared test space, seperated pr --- docs/src/examples/nonlinear_elliptic.md | 4 +- docs/src/tutorials/constraints.md | 2 +- .../tutorials/derivative_neural_network.md | 4 +- src/collocated_estim.jl | 46 - test/BPINN_PDEinvsol_tests.jl | 2397 +---------------- test/BPINN_pde_experimental.jl | 1669 ++++++++++++ test/bpinnexperimental.jl | 140 - 7 files changed, 1675 insertions(+), 2587 deletions(-) delete mode 100644 src/collocated_estim.jl create mode 100644 test/BPINN_pde_experimental.jl delete mode 100644 test/bpinnexperimental.jl diff --git a/docs/src/examples/nonlinear_elliptic.md b/docs/src/examples/nonlinear_elliptic.md index 155330b2bc..d7f8a58579 100644 --- a/docs/src/examples/nonlinear_elliptic.md +++ b/docs/src/examples/nonlinear_elliptic.md @@ -89,7 +89,7 @@ sym_prob = NeuralPDE.symbolic_discretize(pdesystem, discretization) pde_inner_loss_functions = sym_prob.loss_functions.pde_loss_functions bcs_inner_loss_functions = sym_prob.loss_functions.bc_loss_functions[1:6] -aprox_derivative_loss_functions = sym_prob.loss_functions.bc_loss_functions[7:end] +approx_derivative_loss_functions = sym_prob.loss_functions.bc_loss_functions[7:end] global iteration = 0 callback = function (p, l) @@ -97,7 +97,7 @@ callback = function (p, l) println("loss: ", l) println("pde_losses: ", map(l_ -> l_(p.u), pde_inner_loss_functions)) println("bcs_losses: ", map(l_ -> l_(p.u), bcs_inner_loss_functions)) - println("der_losses: ", map(l_ -> l_(p.u), aprox_derivative_loss_functions)) + println("der_losses: ", map(l_ -> l_(p.u), approx_derivative_loss_functions)) end global iteration += 1 return false diff --git a/docs/src/tutorials/constraints.md b/docs/src/tutorials/constraints.md index 0898fab116..e87e047ae3 100644 --- a/docs/src/tutorials/constraints.md +++ b/docs/src/tutorials/constraints.md @@ -74,7 +74,7 @@ sym_prob = NeuralPDE.symbolic_discretize(pdesystem, discretization) pde_inner_loss_functions = sym_prob.loss_functions.pde_loss_functions bcs_inner_loss_functions = sym_prob.loss_functions.bc_loss_functions -aprox_derivative_loss_functions = sym_prob.loss_functions.bc_loss_functions +approx_derivative_loss_functions = sym_prob.loss_functions.bc_loss_functions cb_ = function (p, l) println("loss: ", l) diff --git a/docs/src/tutorials/derivative_neural_network.md b/docs/src/tutorials/derivative_neural_network.md index d7ccec27ad..3963be4308 100644 --- a/docs/src/tutorials/derivative_neural_network.md +++ b/docs/src/tutorials/derivative_neural_network.md @@ -102,13 +102,13 @@ sym_prob = NeuralPDE.symbolic_discretize(pdesystem, discretization) pde_inner_loss_functions = sym_prob.loss_functions.pde_loss_functions bcs_inner_loss_functions = sym_prob.loss_functions.bc_loss_functions[1:7] -aprox_derivative_loss_functions = sym_prob.loss_functions.bc_loss_functions[9:end] +approx_derivative_loss_functions = sym_prob.loss_functions.bc_loss_functions[9:end] callback = function (p, l) println("loss: ", l) println("pde_losses: ", map(l_ -> l_(p.u), pde_inner_loss_functions)) println("bcs_losses: ", map(l_ -> l_(p.u), bcs_inner_loss_functions)) - println("der_losses: ", map(l_ -> l_(p.u), aprox_derivative_loss_functions)) + println("der_losses: ", map(l_ -> l_(p.u), approx_derivative_loss_functions)) return false end diff --git a/src/collocated_estim.jl b/src/collocated_estim.jl deleted file mode 100644 index 0fe608e951..0000000000 --- a/src/collocated_estim.jl +++ /dev/null @@ -1,46 +0,0 @@ -# suggested extra loss function for ODE solver case -function L2loss2(Tar::LogTargetDensity, θ) - f = Tar.prob.f - - # parameter estimation chosen or not - if Tar.extraparams > 0 - autodiff = Tar.autodiff - # Timepoints to enforce Physics - t = Tar.dataset[end] - u1 = Tar.dataset[2] - û = Tar.dataset[1] - - nnsol = NNodederi(Tar, t, θ[1:(length(θ) - Tar.extraparams)], autodiff) - - ode_params = Tar.extraparams == 1 ? - θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : - θ[((length(θ) - Tar.extraparams) + 1):length(θ)] - - if length(Tar.prob.u0) == 1 - physsol = [f(û[i], - ode_params, - t[i]) - for i in 1:length(û[:, 1])] - else - physsol = [f([û[i], u1[i]], - ode_params, - t[i]) - for i in 1:length(û)] - end - #form of NN output matrix output dim x n - deri_physsol = reduce(hcat, physsol) - - physlogprob = 0 - for i in 1:length(Tar.prob.u0) - # can add phystd[i] for u[i] - physlogprob += logpdf(MvNormal(deri_physsol[i, :], - LinearAlgebra.Diagonal(map(abs2, - (Tar.l2std[i] * 4.0) .* - ones(length(nnsol[i, :]))))), - nnsol[i, :]) - end - return physlogprob - else - return 0 - end -end \ No newline at end of file diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index e756a3861f..5cc8fe95fe 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -243,2399 +243,4 @@ u_predict = pmean(sol2.ensemblesol[1]) @test u_predict≈u_real atol=1.5 @test mean(u_predict .- u_real) < 0.1 -@test sol1.estimated_de_params[1]≈param atol=param * 0.3 - - -println("Example 3: Lotka Volterra with New parameter estimation") -@parameters t α β γ δ -@variables x(..) y(..) - -Dt = Differential(t) -eqs = [Dt(x(t)) * α ~ x(t) - β * x(t) * y(t), Dt(y(t)) * δ ~ x(t) * y(t) - y(t)*γ ] -bcs = [x(0) ~ 1.0, y(0) ~ 1.0] -domains = [t ∈ Interval(0.0, 7.0)] - -# Define the parameters' values -# α, β, γ, δ = p - -# regular equations -# dx = (1.5 - y) * x # prey -# dy = (x - 3.0) * y # predator -# p = [1.5, 1.0, 3.0, 1.0] non transformed values - -# transformed equations -# dx*0.666 = (1 - 0.666 * y) * x # prey -# dy*1.0 = (x - 3.0) * y # predator -# p = [0.666, 0.666, 3.0, 1.0] transformed values (change is scale also ensured!) - -chainl = [ - Lux.Chain(Lux.Dense(1, 5, sin), Lux.Dense(5, 5, sin),Lux.Dense(5, 1)), - Lux.Chain(Lux.Dense(1, 5, sin), Lux.Dense(5, 5, sin),Lux.Dense(5, 1)) -] - -initl, st = Lux.setup(Random.default_rng(), chainl[1]) -initl1, st1 = Lux.setup(Random.default_rng(), chainl[2]) - -using NeuralPDE, Lux, OrdinaryDiffEq, Distributions, Random - -function lotka_volterra(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end -# initial-value problem. -u0 = [1.0, 1.0] -# p = [2/3, 2/3, 1/3.0, 1/3.0] -p = [1.5, 1.0, 3.0, 1.0] -tspan = (0.0, 7.0) -prob = ODEProblem(lotka_volterra, u0, tspan, p) -dt = 0.01 -solution = solve(prob, Tsit5(); saveat = dt) -solution1 = solve(prob, Tsit5(); saveat = 0.02) - -function calculate_errors(approx_sol, solution_points) - # Check vector lengths match - if length(approx_sol) != length(solution_points) - error("Vectors must have the same length") - end - - # Calculate errors - n = length(approx_sol) - errors = randn(n) - for i in 1:n - errors[i] = solution_points[i] - approx_sol[i] - end - - # Calculate RMSE - rmse = sqrt(mean(errors.^2)) - - # Calculate MAE - mae = mean(abs.(errors)) - - # Calculate maximum absolute error - max_error = maximum(abs.(errors)) - - # Return dictionary with errors - return Dict( - "RMSE" => rmse, - "MAE" => mae, - "Max Abs Error" => max_error, - ) -end -u = hcat(solution1.u...) -u[1,:] -sol6_2.ensemblesol[1] - -a1=calculate_errors(pmean(sol6_1.ensemblesol[1]), u1[1,:]) -b1=calculate_errors(pmean(sol6_1.ensemblesol[2]), u1[2,:]) - -a=calculate_errors(pmean(sol6_2.ensemblesol[1]), u[1,:]) -b=calculate_errors(pmean(sol6_2.ensemblesol[2]), u[2,:]) - -c=calculate_errors(pmean(sol6_L2_2.ensemblesol[1]), u[1,:]) -d=calculate_errors(pmean(sol6_L2_2.ensemblesol[2]), u[2,:]) - -e=calculate_errors(pmean(sol6_L2_1.ensemblesol[1]), u[1,:]) -f=calculate_errors(pmean(sol6_L2_1.ensemblesol[2]), u[2,:]) - -g=calculate_errors(pmean(sol6_L2.ensemblesol[1]), u[1,:]) -h=calculate_errors(pmean(sol6_L2.ensemblesol[2]), u[2,:]) -sol6_2.ensemblesol[1] -sol6_2.ensemblesol[2] - -sol6_L2.ensemblesol[1] -sol6_L2.ensemblesol[2] - -# function moving_average_smoothing(data::Vector{T}, window_size::Int) where {T} -# smoothed_data = similar(data, T, length(data)) - -# for i in 1:length(data) -# start_idx = max(1, i - window_size) -# end_idx = min(length(data), i + window_size) -# smoothed_data[i] = mean(data[start_idx:end_idx]) -# end - -# return smoothed_data' -# end - -# Extract solution -time = solution.t -u = hcat(solution.u...) -time1=solution.t -u_noisy = u .+ u .* (0.2 .* randn(size(u))) -u_noisy0 = u .+ (3.0 .* rand(size(u)[1],size(u)[2]) .- 1.5) -u_noisy1 = u .+ (0.8.* randn(size(Array(solution)))) -u_noisy2 = u .+ (0.5.* randn(size(Array(solution)))) - -plot(time,u[1,:]) -plot!(time,u[2,:]) -scatter!(time1,u_noisy0[1,:]) -scatter!(time1,u_noisy0[2,:]) -scatter!(discretization_08_gaussian.dataset[1][1][:,2], discretization_08_gaussian.dataset[1][1][:,1]) -scatter!(discretization_08_gaussian.dataset[1][2][:,2], discretization_08_gaussian.dataset[1][2][:,1]) - -scatter!(discretization_05_gaussian.dataset[1][1][:,2], discretization_05_gaussian.dataset[1][1][:,1]) -scatter!(discretization_05_gaussian.dataset[1][2][:,2], discretization_05_gaussian.dataset[1][2][:,1]) -# discretization_05_gaussian.dataset[1][1][:,2] -# window_size = 5 -# smoothed_datasets = [moving_average_smoothing(u1[i, :], window_size) -# for i in 1:length(solution.u[1])] -# u2 = vcat(smoothed_datasets[1], smoothed_datasets[2]) -# Randomly select some points from the solution -num_points = 100 # Number of points to select -selected_indices = rand(1:size(u_noisy1, 2), num_points) -upoints = [u_noisy1[:, i] for i in selected_indices] -timepoints = [time[i] for i in selected_indices] -temp=hcat(upoints...) -dataset = [hcat(temp[i, :], timepoints) for i in 1:2] - -discretization_uniform = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, - dataset = [dataset, nothing]) -discretization_08_gaussian = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, - dataset = [dataset, nothing]) -discretization_05_gaussian = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, - dataset = [dataset, nothing]) - -discretization1 = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, - dataset = [dataset, nothing]) - -scatter!(discretization.dataset[1][1][:,2], discretization.dataset[1][1][:,1]) -scatter!(discretization.dataset[1][2][:,2], discretization.dataset[1][2][:,1]) - -sol = solve(prob, Tsit5(); saveat=0.1) -odedata = Array(sol) + 0.8 * randn(size(Array(sol))) - -@named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [x(t), y(t)], - [α, β, γ, δ], - defaults = Dict([α =>2, β => 2, γ =>2, δ =>2])) - -# creating dictionary for masking equations -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol3 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_uniform = ahmc_bayesian_pinn_pde(pde_system, - discretization_uniform; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_08_gaussian = ahmc_bayesian_pinn_pde(pde_system, - discretization_08_gaussian; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_05_gaussian = ahmc_bayesian_pinn_pde(pde_system, - discretization_05_gaussian; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - - -# more iterations for above -sol3_100_uniform_1000 = ahmc_bayesian_pinn_pde(pde_system, - discretization_uniform; - draw_samples = 1000, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_08_gaussian_1000 = ahmc_bayesian_pinn_pde(pde_system, - discretization_08_gaussian; - draw_samples = 1000, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_05_gaussian_1000 = ahmc_bayesian_pinn_pde(pde_system, - discretization_05_gaussian; - draw_samples = 1000, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - - -# more iterations for above + strict BC -sol3_100_uniform_1000_bc = ahmc_bayesian_pinn_pde(pde_system, - discretization_uniform; - draw_samples = 1000, - bcstd = [0.1, 0.1], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_08_gaussian_1000_bc = ahmc_bayesian_pinn_pde(pde_system, - discretization_08_gaussian; - draw_samples = 1000, - bcstd = [0.1, 0.1], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_08_gaussian_1000_bc_hard = ahmc_bayesian_pinn_pde(pde_system, - discretization_08_gaussian; - draw_samples = 1000, - bcstd = [0.05, 0.05], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_05_gaussian_1000_bc = ahmc_bayesian_pinn_pde(pde_system, - discretization_05_gaussian; - draw_samples = 1000, - bcstd = [0.1, 0.1], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_08_gaussian_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_08_gaussian; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol3_100_05_gaussian_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_05_gaussian; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -# time -# dataset -# chainl[1](time', sol3.estimated_nn_params[1], st)[1][1,:] -# plot!(time1, chainl[1](time1', sol3.estimated_nn_params[1], st)[1][1,:]) -# plot!(time1, chainl[2](time1', sol3.estimated_nn_params[2], st)[1][1,:]) -# plot!(time1, chainl[1](time1', sol5.estimated_nn_params[1], st)[1][1,:]) -# plot!(time1, chainl[2](time1', sol5.estimated_nn_params[2], st)[1][1,:]) -# time1 = collect(0.0:(1 / 100.0):8.0) - -sol4 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true -) - -sol4_0 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true -) - - -sol5_00 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5_0 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - - -# 70 points in dataset -sol6 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -# SOL6_1 VS SOL6_L2 -sol6_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol6_2 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol6_2_L2 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol6_3 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol6_4 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol6_L2 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol6_L2_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol6_L2_2 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.05, 0.05], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -# 50 datapoint 0-5 sol5 vs sol4 -# julia> sol4.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.549 ± 0.0058 -# 0.71 ± 0.0042 -# 0.408 ± 0.0063 -# 0.355 ± 0.0015 - -# julia> sol5.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.604 ± 0.0052 -# 0.702 ± 0.0034 -# 0.346 ± 0.0037 -# 0.335 ± 0.0013 - -# 100 datapoint 0-5 sol5_2 vs sol3 -# julia> sol3.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.598 ± 0.0037 -# 0.711 ± 0.0027 -# 0.399 ± 0.0032 -# 0.333 ± 0.0011 - -# julia> sol5_2.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.604 ± 0.0035 -# 0.686 ± 0.0026 -# 0.395 ± 0.0029 -# 0.328 ± 0.00095 - -# timespan for full dataset (0-8) -sol6 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], progress = true) - -sol5_3 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5_4 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5_5 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], progress = true -) - -sol7 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol5_5_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], progress = true -) - -sol7_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol7_2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.1, 0.1], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol7_3 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol7_4 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -lpfun = function f(chain::Chains) # function to compute the logpdf values - niter, nparams, nchains = size(chain) - lp = zeros(niter + nchains) # resulting logpdf values - for i = 1:nparams - lp += logpdf(MvNormal(Array(chain[:,i,:])) , dataset[1][:,1]') - lp += logpdf(MvNormal(Array(chain[:,i,:])) , dataset[1][:,2]') - end - return lp -end - -DIC, pD = dic(sol3.original.mcmc_chain, lpfun) -DIC1, pD1 = dic(sol4.original.mcmc_chain, lpfun) - -size(sol3.original.mcmc_chain) -Array(sol3.original.mcmc_chain[1,:,:]) -length(sol3.estimated_nn_params[1]) -chainl[1](time', sol3.estimated_nn_params[1], st)[1] - -data = [hcat(calculate_derivatives2(dataset[i][:, 2], dataset[1][:, 1]),dataset[i][:, 2]) for i in eachindex(dataset)] -dataset[1][:,1] -dataset[2] -plot!(dataset[1][:,2],dataset[1][:,1]) -eqs -sol5 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 200, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.02, 0.02], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(3, 2), - Normal(3, 2) - # LogNormal(1, 2), - # LogNormal(1, 2), - # LogNormal(1, 2), - # LogNormal(1, 2) - ], progress = true) - -# plot(time, chainl[1](time', sol2.estimated_nn_params[1], st)[1]) -# plot!(time, chainl[2](time', sol2.estimated_nn_params[2], st)[1]) - -sol6 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 200, - bcstd = [0.5, 0.5], - phystd = [0.5, 0.5], l2std = [0.02, 0.02], - priorsNNw = (0.0, 5.0), phystdnew = [0.5, 0.5], - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8),aa - param = [ - # LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3) - # Normal(3, 2), - # Normal(4, 2), - Normal(3, 2), - Normal(3, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -function calculate_derivatives2(indvar,depvar) - x̂, time = indvar,depvar - num_points = length(x̂) - # Initialize an array to store the derivative values. - derivatives = similar(x̂) - - for i in 2:(num_points - 1) - # Calculate the first-order derivative using central differences. - Δt_forward = time[i + 1] - time[i] - Δt_backward = time[i] - time[i - 1] - - derivative = (x̂[i + 1] - x̂[i - 1]) / (Δt_forward + Δt_backward) - - derivatives[i] = derivative - end - - # Derivatives at the endpoints can be calculated using forward or backward differences. - derivatives[1] = (x̂[2] - x̂[1]) / (time[2] - time[1]) - derivatives[end] = (x̂[end] - x̂[end - 1]) / (time[end] - time[end - 1]) - return derivatives -end -dataset[1] -dataset[2] -dataset[1][:,1]=calculate_derivatives2(dataset[1][:,2], dataset[1][:,1]) -dataset[2][:,1]=calculate_derivatives2(dataset[2][:,2], dataset[2][:,1]) -dataset[1] -dataset[2] -sol7 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 200, - bcstd = [0.5, 0.5], - phystd = [0.5, 0.5], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(0, 2), - Normal(0, 2) - # LogNormal(1, 2), - # LogNormal(1, 2), - # LogNormal(1, 2), - # LogNormal(1, 2) - ], progress = true) - -# plot(time, chainl[1](time', sol2.estimated_nn_params[1], st)[1]) -# plot!(time, chainl[2](time', sol2.estimated_nn_params[2], st)[1]) - -sol8 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), phystdnew = [0.1, 0.1], - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8),aa - param = [ - # LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3),LogNormal(2, 3) - # Normal(3, 2), - # Normal(4, 2), - Normal(0, 2), - Normal(0, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -timepoints = collect(0.0:(1 / 100.0):9.0) -plot!(timepoints', chainl[1](timepoints', sol5_4.estimated_nn_params[1], st)[1]) -plot!(timepoints, chainl[2](timepoints', sol5_4.estimated_nn_params[2], st)[1]) - -sol_L2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true -) - -sol_NEW = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) -sol_L2_70 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true -) - -sol_NEW_70 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -using Plots, StatsPlots -plotly() - -plot(time, u[1, :]) -plot!(time, u[2, :]) -scatter!(time, u_noisy[1, :]) -scatter!(time, u_noisy[2, :]) -scatter!(discretization.dataset[1][1][:,2], discretization.dataset[1][1][:,1]) -scatter!(discretization.dataset[1][2][:,2], discretization.dataset[1][2][:,1]) - -scatter!(discretization1.dataset[1][1][:,2], discretization1.dataset[1][1][:,1],legend=nothing) -scatter!(discretization1.dataset[1][2][:,2], discretization1.dataset[1][2][:,1]) - -# plot28(sol4 seems better vs sol3 plots, params seems similar) -plot!(sol3.timepoints[1]', sol3.ensemblesol[1]) -plot!(sol3.timepoints[2]', sol3.ensemblesol[2]) -plot!(sol3_0.timepoints[1]', sol3_0.ensemblesol[1]) -plot!(sol3_0.timepoints[2]', sol3_0.ensemblesol[2]) - -plot!(sol4.timepoints[1]', sol4.ensemblesol[1]) -plot!(sol4.timepoints[2]', sol4.ensemblesol[2]) -plot!(sol4_0.timepoints[1]', sol4_0.ensemblesol[1]) -plot!(sol4_0.timepoints[2]', sol4_0.ensemblesol[2]) - -plot!(sol4_2.timepoints[1]', sol4_2.ensemblesol[1],legend=nothing) -plot!(sol4_2.timepoints[2]', sol4_2.ensemblesol[2]) -plot!(sol5_2.timepoints[1]', sol5_2.ensemblesol[1],legend=nothing) -plot!(sol5_2.timepoints[2]', sol5_2.ensemblesol[2]) - -plot!(sol4_3.timepoints[1]', sol4_3.ensemblesol[1],legend=nothing) -plot!(sol4_3.timepoints[2]', sol4_3.ensemblesol[2]) -plot!(sol5_3.timepoints[1]', sol5_3.ensemblesol[1]) -plot!(sol5_3.timepoints[2]', sol5_3.ensemblesol[2]) -plot!(sol5_4.timepoints[1]', sol5_4.ensemblesol[1],legend=nothing) -plot!(sol5_4.timepoints[2]', sol5_4.ensemblesol[2]) - - -# plot 36 sol4 vs sol5(params sol4 better, but plots sol5 "looks" better),plot 44(sol5 better than sol6 overall) -plot!(sol5.timepoints[1]', sol5.ensemblesol[1],legend=nothing) -plot!(sol5.timepoints[2]', sol5.ensemblesol[2]) - -plot!(sol5_0.timepoints[1]', sol5_0.ensemblesol[1],legend=nothing) -plot!(sol5_0.timepoints[2]', sol5_0.ensemblesol[2]) - -plot!(sol5_00.timepoints[1]', sol5_00.ensemblesol[1],legend=nothing) -plot!(sol5_00.timepoints[2]', sol5_00.ensemblesol[2]) - -plot!(sol6.timepoints[1]', sol6.ensemblesol[1]) -plot!(sol6.timepoints[2]', sol6.ensemblesol[2]) -plot!(sol6_L2.timepoints[1]', sol6_L2.ensemblesol[1]) -plot!(sol6_L2.timepoints[2]', sol6_L2.ensemblesol[2]) - -plot!(sol6_L2_1.timepoints[1]', sol6_L2_1.ensemblesol[1]) -plot!(sol6_L2_1.timepoints[2]', sol6_L2_1.ensemblesol[2]) - -plot!(sol6_L2_2.timepoints[1]', sol6_L2_2.ensemblesol[1]) -plot!(sol6_L2_2.timepoints[2]', sol6_L2_2.ensemblesol[2]) - -plot!(sol6_1.timepoints[1]', sol6_1.ensemblesol[1]) -plot!(sol6_1.timepoints[2]', sol6_1.ensemblesol[2]) -plot!(sol6_2.timepoints[1]', sol6_2.ensemblesol[1]) -plot!(sol6_2.timepoints[2]', sol6_2.ensemblesol[2],legend=nothing) -plot!(sol6_2_L2.timepoints[1]', sol6_2_L2.ensemblesol[1]) -plot!(sol6_2_L2.timepoints[2]', sol6_2_L2.ensemblesol[2],legend=nothing) - -# plot52 sol7 vs sol5(sol5 overall better plots, params?) -plot!(sol7.timepoints[1]', sol7.ensemblesol[1]) -plot!(sol7.timepoints[2]', sol7.ensemblesol[2]) - -# sol8,sol8_2,sol9,sol9_2 bad -plot!(sol8.timepoints[1]', sol8.ensemblesol[1]) -plot!(sol8.timepoints[2]', sol8.ensemblesol[2]) -plot!(sol8_2.timepoints[1]', sol8_2.ensemblesol[1]) -plot!(sol8_2.timepoints[2]', sol8_2.ensemblesol[2]) - -plot!(sol9.timepoints[1]', sol9.ensemblesol[1]) -plot!(sol9.timepoints[2]', sol9.ensemblesol[2]) -plot!(sol9_2.timepoints[1]', sol9_2.ensemblesol[1]) -plot!(sol9_2.timepoints[2]', sol9_2.ensemblesol[2]) - - -plot!(sol5_5.timepoints[1]', sol5_5.ensemblesol[1]) -plot!(sol5_5.timepoints[2]', sol5_5.ensemblesol[2],legend=nothing) - -plot!(sol5_5_1.timepoints[1]', sol5_5_1.ensemblesol[1]) -plot!(sol5_5_1.timepoints[2]', sol5_5_1.ensemblesol[2],legend=nothing) -plot!(sol7_1.timepoints[1]', sol7_1.ensemblesol[1]) -plot!(sol7_1.timepoints[2]', sol7_1.ensemblesol[2]) - -plot!(sol7_4.timepoints[1]', sol7_4.ensemblesol[1]) -plot!(sol7_4.timepoints[2]', sol7_4.ensemblesol[2]) - -plot!(sol5_2_1.timepoints[1]', sol5_2_1.ensemblesol[1],legend=nothing) -plot!(sol5_2_1.timepoints[2]', sol5_2_1.ensemblesol[2]) -plot!(sol5_2_2.timepoints[1]', sol5_2_2.ensemblesol[1],legend=nothing) -plot!(sol5_2_2.timepoints[2]', sol5_2_2.ensemblesol[2]) - -plot!(sol5_0.timepoints[1]', sol5_0.ensemblesol[1]) -plot!(sol5_0.timepoints[2]', sol5_0.ensemblesol[2],legend=nothing) - -plot!(sol5_00.timepoints[1]', sol5_00.ensemblesol[1],legend=nothing) -plot!(sol5_00.timepoints[2]', sol5_00.ensemblesol[2]) - -plot!(sol3_0.timepoints[1]', sol3_0.ensemblesol[1]) -plot!(sol3_0.timepoints[2]', sol3_0.ensemblesol[2],legend=nothing) -plot!(sol4_0.timepoints[1]', sol4_0.ensemblesol[1]) -plot!(sol4_0.timepoints[2]', sol4_0.ensemblesol[2],legend=nothing) - -plot!(sol3_100_05_gaussian.timepoints[1]', sol3_100_05_gaussian.ensemblesol[1]) -plot!(sol3_100_05_gaussian.timepoints[2]', sol3_100_05_gaussian.ensemblesol[2],legend=nothing) - -plot!(sol3_100_05_gaussian_new.timepoints[1]', sol3_100_05_gaussian_new.ensemblesol[1]) -plot!(sol3_100_05_gaussian_new.timepoints[2]', sol3_100_05_gaussian_new.ensemblesol[2]) - -plot!(sol3_100_08_gaussian.timepoints[1]', sol3_100_08_gaussian.ensemblesol[1]) -plot!(sol3_100_08_gaussian.timepoints[2]', sol3_100_08_gaussian.ensemblesol[2]) - -plot!(sol3_100_08_gaussian_new.timepoints[1]', sol3_100_08_gaussian_new.ensemblesol[1]) -plot!(sol3_100_08_gaussian_new.timepoints[2]', sol3_100_08_gaussian_new.ensemblesol[2],legend=nothing) - -plot!(sol3_100_uniform.timepoints[1]', sol3_100_uniform.ensemblesol[1]) -plot!(sol3_100_uniform.timepoints[2]', sol3_100_uniform.ensemblesol[2]) - -plot!(sol3_100_08_gaussian_1000.timepoints[1]', sol3_100_08_gaussian_1000.ensemblesol[1]) -plot!(sol3_100_08_gaussian_1000.timepoints[2]', sol3_100_08_gaussian_1000.ensemblesol[2]) - -plot!(sol3_100_05_gaussian_1000.timepoints[1]', sol3_100_05_gaussian_1000.ensemblesol[1]) -plot!(sol3_100_05_gaussian_1000.timepoints[2]', sol3_100_05_gaussian_1000.ensemblesol[2]) - -plot!(sol3_100_uniform_1000.timepoints[1]', sol3_100_uniform_1000.ensemblesol[1]) -plot!(sol3_100_uniform_1000.timepoints[2]', sol3_100_uniform_1000.ensemblesol[2]) - -plot!(sol3_100_08_gaussian_1000_bc.timepoints[1]', sol3_100_08_gaussian_1000_bc.ensemblesol[1]) -plot!(sol3_100_08_gaussian_1000_bc.timepoints[2]', sol3_100_08_gaussian_1000_bc.ensemblesol[2]) - -# test with lower number of points -# test same calls 2 times or more -# consider full range dataset case -# combination of all above - -# run 1 100 iters -sol5.estimated_de_params -sol6.estimated_de_params - -# run 2 200 iters -sol5.estimated_de_params -sol6.estimated_de_params - -# run 2 200 iters -sol3.estimated_de_params -sol4.estimated_de_params - -# p = [2/3, 2/3, 1/3, 1/3] -sol3.estimated_de_params -sol4.estimated_de_params -dataset[1] -eqs -α, β, γ, δ = p -p -# 1.0 -# 0.6666666666666666 -# 1.0 -# 0.33333333333333333 - -1/a -1/c -eqs -using StatsPlots -plotly() -plot(sol3.original.mcmc_chain) -plot(sol5_00.original.mcmc_chain) - -# 4-element Vector{Particles{Float64, 34}}: -# 1.23 ± 0.022 -# 0.858 ± 0.011 -# 3.04 ± 0.079 -# 1.03 ± 0.024 -# 4-element Vector{Particles{Float64, 34}}: -# 1.2 ± 0.0069 -# 0.835 ± 0.006 -# 3.22 ± 0.01 -# 1.08 ± 0.0053 -# # plot(time', chainl[1](time', sol1.estimated_nn_params[1], st)[1]) -# # plot!(time, chainl[2](time', sol1.estimated_nn_params[2], st)[1]) - -# sol3 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 500, -# bcstd = [0.05, 0.05], -# phystd = [0.005, 0.005], l2std = [0.1, 0.1], -# phystdnew = [0.5, 0.5], -# # Kernel = AdvancedHMC.NUTS(0.8), -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 50.0], -# param = [ -# Normal(0.0, 2), -# Normal(0.0, 2), -# Normal(0.0, 2), -# Normal(0.0, 2) -# ], -# Dict_differentials = Dict_differentials, progress = true) - -# sol = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 500, -# bcstd = [0.05, 0.05], -# phystd = [0.005, 0.005], l2std = [0.1, 0.1], -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 50.0], -# # Kernel = AdvancedHMC.NUTS(0.8), -# param = [ -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2) -# ], progress = true) - -# plot!(sol.timepoints[1]', sol.ensemblesol[1]) -# plot!(sol.timepoints[2]', sol.ensemblesol[2]) - -# sol1 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 500, -# bcstd = [0.05, 0.05], -# phystd = [0.005, 0.005], l2std = [0.1, 0.1], -# phystdnew = [0.5, 0.5], -# # Kernel = AdvancedHMC.NUTS(0.8), -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 50.0], -# param = [ -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2), -# Normal(1.0, 2) -# ], -# Dict_differentials = Dict_differentials, progress = true) - -# plot!(sol1.timepoints[1]', sol1.ensemblesol[1]) -# plot!(sol1.timepoints[2]', sol1.ensemblesol[2]) - -sol = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 500, - bcstd = [0.05, 0.05], - phystd = [0.005, 0.005], l2std = [0.1, 0.1], - priorsNNw = (0.0, 10.0), - saveats = [1 / 50.0], - # Kernel = AdvancedHMC.NUTS(0.8), - param = [ - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2) - ]) - -# plot!(sol.timepoints[1]', sol.ensemblesol[1]) -# plot!(sol.timepoints[2]', sol.ensemblesol[2]) - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 500, - bcstd = [0.05, 0.05], - phystd = [0.005, 0.005], l2std = [0.1, 0.1], - phystdnew = [0.5, 0.5], - # Kernel = AdvancedHMC.NUTS(0.8), - priorsNNw = (0.0, 10.0), - saveats = [1 / 50.0], - param = [ - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2), - Normal(1.0, 2) - ], - Dict_differentials = Dict_differentials) - -param = 2 * π -ts = vec(sol1.timepoints[1]) -u_real = [analytic_sol_func1(0.0, t) for t in ts] -u_predict = pmean(sol1.ensemblesol[1]) - -@test u_predict≈u_real atol=1.5 -@test mean(u_predict .- u_real) < 0.1 -@test sol1.estimated_de_params[1]≈param atol=param * 0.3 - -# points1 = [] -# for eq_arg in eq_args -# a = [] -# # for each (depvar,[indvar1..]) if indvari==indvar (eq_arg) -# for i in eachindex(symbols_input) -# if symbols_input[i][2] == eq_arg -# # include domain points of that depvar -# # each loss equation take domain matrix [points..;points..] -# push!(a, train_sets[i][:, 2:end]') -# end -# end -# # vcat as new row for next equation -# push!(points1, vcat(a...)) -# end -# println(points1 == points) - -# using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC -# import ModelingToolkit: Interval, infimum, supremum, Distributions -# using Plots, MonteCarloMeasurements - -# @parameters x, t, α -# @variables u(..) -# Dt = Differential(t) -# Dx = Differential(x) -# Dx2 = Differential(x)^2 -# Dx3 = Differential(x)^3 -# Dx4 = Differential(x)^4 - -# # α = 1 -# β = 4 -# γ = 1 -# eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 - -# u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -# du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 - -# bcs = [u(x, 0) ~ u_analytic(x, 0), -# u(-10, t) ~ u_analytic(-10, t), -# u(10, t) ~ u_analytic(10, t), -# Dx(u(-10, t)) ~ du(-10, t), -# Dx(u(10, t)) ~ du(10, t)] - -# # Space and time domains -# domains = [x ∈ Interval(-10.0, 10.0), -# t ∈ Interval(0.0, 1.0)] - -# # Discretization -# dx = 0.4; -# dt = 0.2; - -# # Function to compute analytical solution at a specific point (x, t) -# function u_analytic_point(x, t) -# z = -x / 2 + t -# return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -# end - -# # Function to generate the dataset matrix -# function generate_dataset_matrix(domains, dx, dt) -# x_values = -10:dx:10 -# t_values = 0.0:dt:1.0 - -# dataset = [] - -# for t in t_values -# for x in x_values -# u_value = u_analytic_point(x, t) -# push!(dataset, [u_value, x, t]) -# end -# end - -# return vcat([data' for data in dataset]...) -# end - -# datasetpde = [generate_dataset_matrix(domains, dx, dt)] - -# # noise to dataset -# noisydataset = deepcopy(datasetpde) -# noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ -# randn(size(noisydataset[1][:, 1])) .* 5 / 100 .* -# noisydataset[1][:, 1] - -# # plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") -# # plot!(noisydataset[1][:, 2], noisydataset[1][:, 1]) - -# # Neural network -# chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), -# Lux.Dense(8, 8, Lux.tanh), -# Lux.Dense(8, 1)) - -# discretization = NeuralPDE.BayesianPINN([chain], -# GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) - -# @named pde_system = PDESystem(eq, -# bcs, -# domains, -# [x, t], -# [u(x, t)], -# [α], -# defaults = Dict([α => 0.5])) - -# sol1 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 100, -# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], -# phystd = [1.0], l2std = [0.05], param = [Distributions.LogNormal(0.5, 2)], -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 100.0, 1 / 100.0], progress = true) - -# eqs = pde_system.eqs -# Dict_differentials = Dict() -# exps = toexpr.(eqs) -# nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -# sol2 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 100, -# bcstd = [0.2, 0.2, 0.2, 0.2, 0.2], -# phystd = [1.0], phystdnew = [0.05], l2std = [0.05], -# param = [Distributions.LogNormal(0.5, 2)], -# priorsNNw = (0.0, 10.0), -# saveats = [1 / 100.0, 1 / 100.0], Dict_differentials = Dict_differentials, -# progress = true) - -# phi = discretization.phi[1] -# xs, ts = [infimum(d.domain):dx:supremum(d.domain) -# for (d, dx) in zip(domains, [dx / 10, dt])] -# u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] -# for t in ts] -# u_real = [[u_analytic(x, t) for x in xs] for t in ts] -# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) -# for x in xs] -# for t in ts] - -# # p1 = plot(xs, u_predict, title = "predict") -# # p2 = plot(xs, u_real, title = "analytic") -# # p3 = plot(xs, diff_u, title = "error") -# # plot(p1, p2, p3) - -# phi = discretization.phi[1] -# xs, ts = [infimum(d.domain):dx:supremum(d.domain) -# for (d, dx) in zip(domains, [dx / 10, dt])] -# u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] -# for t in ts] -# u_real = [[u_analytic(x, t) for x in xs] for t in ts] -# diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) -# for x in xs] -# for t in ts] - -# # p1 = plot(xs, u_predict, title = "predict") -# # p2 = plot(xs, u_real, title = "analytic") -# # p3 = plot(xs, diff_u, title = "error") -# # plot(p1, p2, p3) - -@parameters t, p -@variables u(..) - -Dt = Differential(t) -eqs = Dt(u(t)) - cos(p * t) ~ 0 -bcs = [u(0) ~ 0.0] -domains = [t ∈ Interval(0.0, 2.0)] - -chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) -initl, st = Lux.setup(Random.default_rng(), chainl) - -@named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [u(t)], - [p], - defaults = Dict([p => 4.0])) - -analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) -timepoints = collect(0.0:(1 / 100.0):2.0) -u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] -u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) -dataset = [hcat(u1, timepoints)] - -discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, - dataset = [dataset, nothing]) - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(4.0, 2)], progress = true) - -param = 2 * π -ts = vec(sol1.timepoints[1]) -u_real = [analytic_sol_func1(0.0, t) for t in ts] -u_predict = pmean(sol1.ensemblesol[1]) - -@test u_predict≈u_real atol=0.1 -@test mean(u_predict .- u_real) < 0.01 -@test sol1.estimated_de_params[1]≈param atol=0.1 -sol1.estimated_de_params[1] - -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.02], phystdnew = [0.02], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(4.0, 2)], - Dict_differentials = Dict_differentials, - progress = true) - -param = 2 * π -ts_2 = vec(sol2.timepoints[1]) -u_real_2 = [analytic_sol_func1(0.0, t) for t in ts] -u_predict_2 = pmean(sol2.ensemblesol[1]) - -@test u_predict_2≈u_real_2 atol=0.1 -@test mean(u_predict_2 .- u_real_2) < 0.01 -@test sol2.estimated_de_params[1]≈param atol=0.1 -sol2.estimated_de_params[1] - -plot(ts_2, u_predict_2) -plot!(ts_2, u_real_2) - -@parameters t, σ_ -@variables x(..), y(..), z(..) -Dt = Differential(t) -eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), - Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), - Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] - -bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] -domains = [t ∈ Interval(0.0, 1.0)] - -input_ = length(domains) -n = 7 -chain = [ - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)) -] - -#Generate Data -function lorenz!(du, u, p, t) - du[1] = 10.0 * (u[2] - u[1]) - du[2] = u[1] * (28.0 - u[3]) - u[2] - du[3] = u[1] * u[2] - (8 / 3) * u[3] -end - -u0 = [1.0; 0.0; 0.0] -tspan = (0.0, 1.0) -prob = ODEProblem(lorenz!, u0, tspan) -sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) -ts = sol.t -us = hcat(sol.u...) -us = us .+ ((0.05 .* randn(size(us))) .* us) -ts_ = hcat(sol(ts).t...)[1, :] -dataset = [hcat(us[i, :], ts_) for i in 1:3] - -discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, - dataset = [dataset, nothing]) - -@named pde_system = PDESystem(eqs, bcs, domains, - [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 100, - bcstd = [0.3, 0.3, 0.3], - phystd = [0.1, 0.1, 0.1], - l2std = [1, 1, 1], - priorsNNw = (0.0, 1.0), - saveats = [0.01], - param = [Normal(14.0, 2)], progress = true) - -idealp = 10.0 -p_ = sol1.estimated_de_params[1] -@test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] -# @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] - -@parameters x y -@variables u(..) -Dxx = Differential(x)^2 -Dyy = Differential(y)^2 - -# 2D PDE -eq = Dxx(u(x, y)) + Dyy(u(x, y)) ~ -sin(pi * x) * sin(pi * y) - -# Boundary conditions -bcs = [u(0, y) ~ 0.0, u(1, y) ~ 0.0, - u(x, 0) ~ 0.0, u(x, 1) ~ 0.0] - -# Space and time domains -domains = [x ∈ Interval(0.0, 1.0), - y ∈ Interval(0.0, 1.0)] - -# Neural network -dim = 2 # number of dimensions -chain = Lux.Chain(Lux.Dense(dim, 9, Lux.σ), Lux.Dense(9, 9, Lux.σ), Lux.Dense(9, 1)) - -# Discretization -dx = 0.04 -discretization = BayesianPINN([chain], GridTraining(dx), dataset = [[dataset], nothing]) - -@named pde_system = PDESystem(eq, bcs, domains, [x, y], [u(x, y)]) - -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 5, - bcstd = [0.01, 0.01, 0.01, 0.01], - phystd = [0.005], - priorsNNw = (0.0, 2.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -xs = sol1.timepoints[1] -sol1.ensemblesol[1] -analytic_sol_func(x, y) = (sin(pi * x) * sin(pi * y)) / (2pi^2) - -dataset = hcat(u_real, xs') -u_predict = pmean(sol1.ensemblesol[1]) -u_real = [analytic_sol_func(xs[:, i][1], xs[:, i][2]) for i in 1:length(xs[1, :])] -@test u_predict≈u_real atol=0.8 - -using NeuralPDE, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC -import ModelingToolkit: Interval, infimum, supremum, Distributions -using Plots, MonteCarloMeasurements - -@parameters x t -@variables u(..) - -Dt = Differential(t) -Dx = Differential(x) -Dxx = Dx^2 -α = 0.05 -# Burger's equation -eq = Dt(u(t, x)) + u(t, x) * Dx(u(t, x)) - α * Dxx(u(t, x)) ~ 0 - -# boundary conditions -bcs = [ - u(0.0, x) ~ -sin(π * x), - u(t, -1.0) ~ 0.0, - u(t, 1.0) ~ 0.0 -] - -domains = [t ∈ Interval(0.0, 1.0), x ∈ Interval(-1.0, 1.0)] - -# Neural network -chain = Lux.Chain(Dense(2, 10, Lux.σ), Dense(10, 10, Lux.σ), Dense(10, 1)) -strategy = NeuralPDE.QuadratureTraining(; abstol = 1e-6, reltol = 1e-6, batch = 200) - -indvars = [t, x] -depvars = [u(t, x)] -@named pde_system = PDESystem(eq, bcs, domains, indvars, depvars) - -# KS EQUATION -using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC -import ModelingToolkit: Interval, infimum, supremum, Distributions -using Plots, MonteCarloMeasurements, StatsPlots - -@parameters x, t, α -@variables u(..) -Dt = Differential(t) -Dx = Differential(x) -Dx2 = Differential(x)^2 -Dx3 = Differential(x)^3 -Dx4 = Differential(x)^4 - -# α = 1 -β = 4 -γ = 1 -eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 - -u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 - -bcs = [u(x, 0) ~ u_analytic(x, 0), - u(-10, t) ~ u_analytic(-10, t), - u(10, t) ~ u_analytic(10, t), - Dx(u(-10, t)) ~ du(-10, t), - Dx(u(10, t)) ~ du(10, t)] - -# Space and time domains -domains = [x ∈ Interval(-10.0, 10.0), - t ∈ Interval(0.0, 1.0)] - -# Discretization -dx = 0.4; -dt = 0.2; - -# Function to compute analytical solution at a specific point (x, t) -function u_analytic_point(x, t) - z = -x / 2 + t - return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -end - -# Function to generate the dataset matrix -function generate_dataset_matrix(domains, dx, dt, xlim, tlim) - x_values = xlim[1]:dx:xlim[2] - t_values = tlim[1]:dt:tlim[2] - - dataset = [] - - for t in t_values - for x in x_values - u_value = u_analytic_point(x, t) - push!(dataset, [u_value, x, t]) - end - end - - return vcat([data' for data in dataset]...) -end - -# x_values = -10:dx:10 -# t_values = 0.0:dt:1.0 - -# dataset = [] - -# for t in t_values -# for x in x_values -# u_value = u_analytic_point(x, t) -# push!(dataset, [u_value, x, t]) -# end -# end -# dataset -# pop= vcat([data' for data in dataset]...) - -datasetpde = [generate_dataset_matrix(domains, dx, dt, [-10,10], [0.0,1.0])] - -datasetpde_new = [generate_dataset_matrix(domains, dx, dt, [-10,0], [0.0,1.0])] - -# noise to dataset -noisydataset = deepcopy(datasetpde) -noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ (randn(size(noisydataset[1][:, 1])) .* 0.8) - -noisydataset_new = deepcopy(datasetpde_new) -noisydataset_new[1][:, 1] = noisydataset_new[1][:, 1] .+ (randn(size(noisydataset_new[1][:, 1])) .* 0.8) - -# a=discretization_new.dataset[1] - -plotly() -plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") -# scatter!(a[1][:, 2], a[1][:, 1]) -scatter!(noisydataset[1][:, 2], noisydataset[1][:, 1]) - -plot(datasetpde[1][:, 2],datasetpde[1][:, 3], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") -scatter!(noisydataset[1][:, 2],noisydataset[1][:, 3], noisydataset[1][:, 1]) - -plotly() -plot(datasetpde_new[1][:, 2], datasetpde_new[1][:, 1], title = "Dataset from Analytical Solution") -scatter!(noisydataset_new[1][:, 2], noisydataset_new[1][:, 1]) - -plot(datasetpde_new[1][:, 2],datasetpde_new[1][:, 3], datasetpde_new[1][:, 1], title = "Dataset from Analytical Solution") -scatter!(noisydataset_new[1][:, 2],noisydataset_new[1][:, 3], noisydataset_new[1][:, 1]) - -noise_std = 1.4 -original_data = datasetpde[1][:, 1] -original_std = std(original_data) -ratio = noise_std / original_std - - -using StatsPlots -plot(sol1.original.mcmc_chain) -plot(sol2.original.mcmc_chain) - -plot(sol0_new.original.mcmc_chain) -plot(sol2_new.original.mcmc_chain) - -# Neural network -chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), - Lux.Dense(8, 8, Lux.tanh), - Lux.Dense(8, 1)) - -chain_more = Lux.Chain(Lux.Dense(2, 10, Lux.tanh), - Lux.Dense(10, 10, Lux.tanh), - Lux.Dense(10, 1)) -# chain = Lux.Chain(Lux.Dense(2, 8, Lux.σ), -# Lux.Dense(8, 8, Lux.σ), -# Lux.Dense(8, 1)) - -discretization = NeuralPDE.BayesianPINN([chain], - GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) - -discretization_more = NeuralPDE.BayesianPINN([chain_more], - GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) - -discretization_new = NeuralPDE.BayesianPINN([chain], - GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset_new, nothing]) - - -@named pde_system = PDESystem(eq, - bcs, - domains, - [x, t], - [u(x, t)], - [α], - defaults = Dict([α => 2.0])) - -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -Dict_differentials - -plot(sol1.original.mcmc_chain) -meanplot(sol1.original.mcmc_chain) -autocorplot(sol1.original.mcmc_chain) -traceplot(sol1.original.mcmc_chain) - -plot(sol2.original.mcmc_chain) -meanplot(sol2.original.mcmc_chain) -autocorplot(sol2.original.mcmc_chain) -traceplot(sol2.original.mcmc_chain) - -plot(sol0_new.original.mcmc_chain) -meanplot(sol0_new.original.mcmc_chain) -autocorplot(sol0_new.original.mcmc_chain) - -plot(sol2_new.original.mcmc_chain) -meanplot(sol2_new.original.mcmc_chain) -autocorplot(sol2_new.original.mcmc_chain) - -plot(sol3_new.original.mcmc_chain) -meanplot(sol3_new.original.mcmc_chain) -autocorplot(sol3_new.original.mcmc_chain) - - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.5], - phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol1_more = ahmc_bayesian_pinn_pde(pde_system, - discretization_more; - draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.5], - phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol1_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 90, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.7], - phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -sol2_more = ahmc_bayesian_pinn_pde(pde_system, - discretization_more; - draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -# julia> sol2 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 85, Kernel = AdvancedHMC.NUTS(0.8), -# bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], -# phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, -# priorsNNw = (0.0, 3.0), -# saveats = [1 / 100.0, 1 / 100.0], -# progress = true) -# ┌ Info: Current Physics Log-likelihood : -# └ ℓπ.full_loglikelihood(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -415167 -# ┌ Info: Current Prior Log-likelihood : -# └ priorlogpdf(ℓπ, initial_θ) = -214.1825373360679 -# ┌ Info: Current MSE against dataset Log-likelihood : -# └ L2LossData(ℓπ, initial_θ) = -109309.44973223892 -# Sampling 100%|███████████████████████████████| Time: 0:14:50 -# iterations: 85 -# ratio_divergent_transitions: 0.0 -# ratio_divergent_transitions_during_adaption: 0.02 -# n_steps: 127 -# is_accept: true -# acceptance_rate: 0.9982795867682919 -# log_density: -3832.934953640867 -# hamiltonian_energy: 4145.005901868316 -# hamiltonian_energy_error: -0.07863051782624098 -# max_hamiltonian_energy_error: -0.16790754244266282 -# tree_depth: 7 -# numerical_error: false -# step_size: 0.00018186972987192408 -# nom_step_size: 0.00018186972987192408 -# is_adapt: false -# mass_matrix: DiagEuclideanMetric([1.0, 1. -# [ Info: Sampling Complete. -# ┌ Info: Current Physics Log-likelihood : -# └ ℓπ.full_loglikelihood(setparameters(ℓπ, samples[end]), ℓπ.allstd) = -132 -# ┌ Info: Current Prior Log-likelihood : -# └ priorlogpdf(ℓπ, samples[end]) = -219.17544656823006 -# ┌ Info: Current MSE against dataset Log-likelihood : -# └ L2LossData(ℓπ, samples[end]) = -3481.509412470054 - -# julia> sol1 = ahmc_bayesian_pinn_pde(pde_system, -# discretization; -# draw_samples = 90, Kernel = AdvancedHMC.NUTS(0.8), -# bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.7], -# phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, -# priorsNNw = (0.0, 3.0), -# saveats = [1 / 100.0, 1 / 100.0], -# Dict_differentials = Dict_differentials, -# progress = true) -# ┌ Info: Current Physics Log-likelihood : -# └ ℓπ.full_loglikelihood(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -394622 -# ┌ Info: Current Prior Log-likelihood : -# └ priorlogpdf(ℓπ, initial_θ) = -214.1657203956881 -# ┌ Info: Current MSE against dataset Log-likelihood : -# └ L2LossData(ℓπ, initial_θ) = -107600.2750860966 -# ┌ Info: Current L2_LOSSY : -# └ ℓπ.L2_loss2(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -352.339686469935 -# Sampling 100%|███████████████████████████████| Time: 0:38:42 -# iterations: 90 -# ratio_divergent_transitions: 0.24 -# ratio_divergent_transitions_during_adaption: 0.02 -# n_steps: 34 -# is_accept: true -# acceptance_rate: 0.0755469536430885 -# log_density: -6535.135018473582 -# hamiltonian_energy: 6681.540376258076 -# hamiltonian_energy_error: -1.7097735125544204 -# max_hamiltonian_energy_error: 1216.239238705054 -# tree_depth: 5 -# numerical_error: true -# step_size: 0.0004111092751764056 -# nom_step_size: 0.0004111092751764056 -# is_adapt: false -# mass_matrix: DiagEuclideanMetric([1.0, 1. -# [ Info: Sampling Complete. -# ┌ Info: Current Physics Log-likelihood : -# └ ℓπ.full_loglikelihood(setparameters(ℓπ, samples[end]), ℓπ.allstd) = -272 -# ┌ Info: Current Prior Log-likelihood : -# └ priorlogpdf(ℓπ, samples[end]) = -218.6535874132563 -# ┌ Info: Current MSE against dataset Log-likelihood : -# └ L2LossData(ℓπ, samples[end]) = -3573.449092586736 -# ┌ Info: Current L2_LOSSY : -# └ ℓπ.L2_loss2(setparameters(ℓπ, samples[end]), ℓπ.allstd) = -2470.35523478 - -using MCMCChains -println(summarize(sol1.original.mcmc_chain)) -plot(sol1.original.mcmc_chain) - -sol3 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 100, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.7], l2std = [0.15], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 3.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -sol4 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1_1.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_1.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol3.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol3.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol4.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol4.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -sol0_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], - phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - - -julia> sol5_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 170, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -sol1_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], - phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol1_1_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.5], - phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol1_2_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 150, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.1], - phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 3.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol1_3_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 150, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], - phystd = [0.3], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 3.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol2_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 140, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - - -sol3_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 140, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -sol4_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 160, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -sol5_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 170, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -# phi = discretization.phi[1] - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol0_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol0_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1_1_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_1_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1_2_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_2_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1_3_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_3_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol2_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol3_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol3_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol4_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol4_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol5_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol5_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol5_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol5_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] -plotly() -p1 = plot(ts, xs, u_predict, title = "predict") -p2 = plot(ts, xs, u_real, title = "analytic") -p3 = plot(ts, xs, diff_u, title = "error") -plot(p1, p2, p3) -# julia> sol0_new = ahmc_bayesian_pinn_pde(pde_system, -# discretization_new; -# draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), -# bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], -# phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], -# priorsNNw = (0.0, 1.0), -# saveats = [1 / 100.0, 1 / 100.0], -# Dict_differentials = Dict_differentials, -# progress = true) -# ┌ Info: Current Physics Log-likelihood : -# └ ℓπ.full_loglikelihood(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -398314.38213382766 -# ┌ Info: Current Prior Log-likelihood : -# └ priorlogpdf(ℓπ, initial_θ) = -104.7365701596561 -# ┌ Info: Current MSE against dataset Log-likelihood : -# └ L2LossData(ℓπ, initial_θ) = -58553.36940699288 -# ┌ Info: Current L2_LOSSY : -# └ ℓπ.L2_loss2(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -281.85131447737575 -# Sampling 100%|███████████████████████████████| Time: 0:26:00 -# iterations: 110 -# ratio_divergent_transitions: 0.2 -# ratio_divergent_transitions_during_adaption: 0.03 -# n_steps: 11 -# is_accept: true -# acceptance_rate: 0.0024891070448310416 -# log_density: -13158.729119075539 -# hamiltonian_energy: 13212.763613683248 -# hamiltonian_energy_error: 0.0 -# max_hamiltonian_energy_error: 1492.7356803165876 -# tree_depth: 3 -# numerical_error: true -# step_size: 0.0002145156661425442 -# nom_step_size: 0.0002145156661425442 -# is_adapt: false -# mass_matrix: DiagEuclideanMetric([1.0, 1.0, 1.0, 1.0, 1.0, 1 ...]) -# [ Info: Sampling Complete. -# ┌ Info: Current Physics Log-likelihood : -# └ ℓπ.full_loglikelihood(setparameters(ℓπ, samples[end]), ℓπ.allstd) = -908.7769621441158 -# ┌ Info: Current Prior Log-likelihood : -# └ priorlogpdf(ℓπ, samples[end]) = -136.87645881663929 -# ┌ Info: Current MSE against dataset Log-likelihood : -# └ L2LossData(ℓπ, samples[end]) = -1404.7102059521355 -# ┌ Info: Current L2_LOSSY : -# └ ℓπ.L2_loss2(setparameters(ℓπ, samples[end]), ℓπ.allstd) = -10708.363203924739 - -# julia> sol2_new = ahmc_bayesian_pinn_pde(pde_system, -# discretization_new; -# draw_samples = 140, Kernel = AdvancedHMC.NUTS(0.8), -# bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], -# phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], -# priorsNNw = (0.0, 1.0), -# saveats = [1 / 100.0, 1 / 100.0], -# progress = true) -# ┌ Info: Current Physics Log-likelihood : -# └ ℓπ.full_loglikelihood(setparameters(ℓπ, initial_θ), ℓπ.allstd) = -397526.19267355377 -# ┌ Info: Current Prior Log-likelihood : -# └ priorlogpdf(ℓπ, initial_θ) = -105.03439044100367 -# ┌ Info: Current MSE against dataset Log-likelihood : -# └ L2LossData(ℓπ, initial_θ) = -60957.24454333089 -# Sampling 99%|███████████████████████████████| ETA: 0:00:10 -# iterations: 140 -# ratio_divergent_transitions: 0.0 -# ratio_divergent_transitions_during_adaption: 0.01 -# n_steps: 1023 -# is_accept: true -# acceptance_rate: 0.972620625460237 -# log_density: -1513.1769839294327 -# hamiltonian_energy: 8709.204139640105 -# hamiltonian_energy_error: -0.4925547801958601 -# max_hamiltonian_energy_error: -1.7861646674082294 -# tree_depth: 10 -# numerical_error: false -# step_size: 0.00011428277138492957 -# nom_step_size: 0.00011428277138492957 -# is_adapt: false -# mass_matrix: DiagEuclideanMetric([1.0, 1.0, 1.0, 1.0, 1.0, 1 ...]) -# [ Info: Sampling Complete. -# ┌ Info: Current Physics Log-likelihood : -# └ ℓπ.full_loglikelihood(setparameters(ℓπ, samples[end]), ℓπ.allstd) = 115.103823132341 -# ┌ Info: Current Prior Log-likelihood : -# └ priorlogpdf(ℓπ, samples[end]) = -198.39103020815858 -# ┌ Info: Current MSE against dataset Log-likelihood : -# └ L2LossData(ℓπ, samples[end]) = -1429.7843027541815 +@test sol1.estimated_de_params[1]≈param atol=param * 0.3 \ No newline at end of file diff --git a/test/BPINN_pde_experimental.jl b/test/BPINN_pde_experimental.jl new file mode 100644 index 0000000000..a8f4a0341e --- /dev/null +++ b/test/BPINN_pde_experimental.jl @@ -0,0 +1,1669 @@ +using Test, MCMCChains, Lux, ModelingToolkit +import ModelingToolkit: Interval, infimum, supremum +using ForwardDiff, Distributions, OrdinaryDiffEq +using AdvancedHMC, Statistics, Random, Functors +using NeuralPDE, MonteCarloMeasurements +using ComponentArrays, ModelingToolkit + +Random.seed!(100) + +# function required to use the new loss, creates a dicitonary of differntial operator terms +function recur_expression(exp, Dict_differentials) + for in_exp in exp.args + if !(in_exp isa Expr) + # skip +,== symbols, characters etc + continue + + elseif in_exp.args[1] isa ModelingToolkit.Differential + # first symbol of differential term + # Dict_differentials for masking differential terms + # and resubstituting differentials in equations after putting in interpolations + # temp = in_exp.args[end] + Dict_differentials[eval(in_exp)] = Symbolics.variable("diff_$(length(Dict_differentials) + 1)") + return + else + recur_expression(in_exp, Dict_differentials) + end + end +end + +# experiments are here +println("Example 3: Lotka Volterra with New parameter estimation") +@parameters t α β γ δ +@variables x(..) y(..) + +Dt = Differential(t) +eqs = [Dt(x(t)) * α ~ x(t) - β * x(t) * y(t), Dt(y(t)) * δ ~ x(t) * y(t) - y(t) * γ] +bcs = [x(0) ~ 1.0, y(0) ~ 1.0] +domains = [t ∈ Interval(0.0, 7.0)] + +# Define the parameters' values +# α, β, γ, δ = p + +# regular equations +# dx = (1.5 - y) * x # prey +# dy = (x - 3.0) * y # predator +# p = [1.5, 1.0, 3.0, 1.0] non transformed values + +# transformed equations +# dx*0.666 = (1 - 0.666 * y) * x # prey +# dy*1.0 = (x - 3.0) * y # predator +# p = [0.666, 0.666, 3.0, 1.0] transformed values (change is scale also ensured!) + +chainl = [ + Lux.Chain(Lux.Dense(1, 5, sin), Lux.Dense(5, 5, sin), Lux.Dense(5, 1)), + Lux.Chain(Lux.Dense(1, 5, sin), Lux.Dense(5, 5, sin), Lux.Dense(5, 1)) +] + +initl, st = Lux.setup(Random.default_rng(), chainl[1]) +initl1, st1 = Lux.setup(Random.default_rng(), chainl[2]) + +using NeuralPDE, Lux, OrdinaryDiffEq, Distributions, Random + +function lotka_volterra(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (α - β * y) * x # prey + dy = (δ * x - γ) * y # predator + + return [dx, dy] +end +# initial-value problem. +u0 = [1.0, 1.0] +# p = [2/3, 2/3, 1/3.0, 1/3.0] +p = [1.5, 1.0, 3.0, 1.0] +tspan = (0.0, 7.0) +prob = ODEProblem(lotka_volterra, u0, tspan, p) +dt = 0.01 +solution = solve(prob, Tsit5(); saveat = dt) +solution1 = solve(prob, Tsit5(); saveat = 0.02) + +function calculate_errors(approx_sol, solution_points) + # Check vector lengths match + if length(approx_sol) != length(solution_points) + error("Vectors must have the same length") + end + + # Calculate errors + n = length(approx_sol) + errors = randn(n) + for i in 1:n + errors[i] = solution_points[i] - approx_sol[i] + end + + # Calculate RMSE + rmse = sqrt(mean(errors .^ 2)) + + # Calculate MAE + mae = mean(abs.(errors)) + + # Calculate maximum absolute error + max_error = maximum(abs.(errors)) + + # Return dictionary with errors + return Dict( + "RMSE" => rmse, + "MAE" => mae, + "Max Abs Error" => max_error + ) +end +u = hcat(solution1.u...) + +a1 = calculate_errors(pmean(sol6_1.ensemblesol[1]), u1[1, :]) +b1 = calculate_errors(pmean(sol6_1.ensemblesol[2]), u1[2, :]) + +a = calculate_errors(pmean(sol6_2.ensemblesol[1]), u[1, :]) +b = calculate_errors(pmean(sol6_2.ensemblesol[2]), u[2, :]) + +c = calculate_errors(pmean(sol6_L2_2.ensemblesol[1]), u[1, :]) +d = calculate_errors(pmean(sol6_L2_2.ensemblesol[2]), u[2, :]) + +e = calculate_errors(pmean(sol6_L2_1.ensemblesol[1]), u[1, :]) +f = calculate_errors(pmean(sol6_L2_1.ensemblesol[2]), u[2, :]) + +g = calculate_errors(pmean(sol6_L2.ensemblesol[1]), u[1, :]) +h = calculate_errors(pmean(sol6_L2.ensemblesol[2]), u[2, :]) + +# function moving_average_smoothing(data::Vector{T}, window_size::Int) where {T} +# smoothed_data = similar(data, T, length(data)) + +# for i in 1:length(data) +# start_idx = max(1, i - window_size) +# end_idx = min(length(data), i + window_size) +# smoothed_data[i] = mean(data[start_idx:end_idx]) +# end + +# return smoothed_data' +# end + +# Extract solution +time = solution.t +u = hcat(solution.u...) +time1 = solution.t +u_noisy = u .+ u .* (0.2 .* randn(size(u))) +u_noisy0 = u .+ (3.0 .* rand(size(u)[1], size(u)[2]) .- 1.5) +u_noisy1 = u .+ (0.8 .* randn(size(Array(solution)))) +u_noisy2 = u .+ (0.5 .* randn(size(Array(solution)))) + +plot(time, u[1, :]) +plot!(time, u[2, :]) +scatter!(time1, u_noisy0[1, :]) +scatter!(time1, u_noisy0[2, :]) +scatter!(discretization_08_gaussian.dataset[1][1][:, 2], + discretization_08_gaussian.dataset[1][1][:, 1]) +scatter!(discretization_08_gaussian.dataset[1][2][:, 2], + discretization_08_gaussian.dataset[1][2][:, 1]) + +scatter!(discretization_05_gaussian.dataset[1][1][:, 2], + discretization_05_gaussian.dataset[1][1][:, 1]) +scatter!(discretization_05_gaussian.dataset[1][2][:, 2], + discretization_05_gaussian.dataset[1][2][:, 1]) +# discretization_05_gaussian.dataset[1][1][:,2] +# window_size = 5 +# smoothed_datasets = [moving_average_smoothing(u1[i, :], window_size) +# for i in 1:length(solution.u[1])] +# u2 = vcat(smoothed_datasets[1], smoothed_datasets[2]) + +# Randomly select some points from the solution +num_points = 100 # Number of points to select +selected_indices = rand(1:size(u_noisy1, 2), num_points) +upoints = [u_noisy1[:, i] for i in selected_indices] +timepoints = [time[i] for i in selected_indices] +temp = hcat(upoints...) +dataset = [hcat(temp[i, :], timepoints) for i in 1:2] + +discretization_uniform = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, + dataset = [dataset, nothing]) +discretization_08_gaussian = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, + dataset = [dataset, nothing]) +discretization_05_gaussian = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, + dataset = [dataset, nothing]) + +discretization1 = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, + dataset = [dataset, nothing]) + +scatter!(discretization.dataset[1][1][:, 2], discretization.dataset[1][1][:, 1]) +scatter!(discretization.dataset[1][2][:, 2], discretization.dataset[1][2][:, 1]) + +sol = solve(prob, Tsit5(); saveat = 0.1) +odedata = Array(sol) + 0.8 * randn(size(Array(sol))) + +@named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [x(t), y(t)], + [α, β, γ, δ], + defaults = Dict([α => 2, β => 2, γ => 2, δ => 2])) + +# creating dictionary for masking equations +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol3 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_uniform = ahmc_bayesian_pinn_pde(pde_system, + discretization_uniform; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_08_gaussian = ahmc_bayesian_pinn_pde(pde_system, + discretization_08_gaussian; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_05_gaussian = ahmc_bayesian_pinn_pde(pde_system, + discretization_05_gaussian; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +# more iterations for above +sol3_100_uniform_1000 = ahmc_bayesian_pinn_pde(pde_system, + discretization_uniform; + draw_samples = 1000, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_08_gaussian_1000 = ahmc_bayesian_pinn_pde(pde_system, + discretization_08_gaussian; + draw_samples = 1000, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_05_gaussian_1000 = ahmc_bayesian_pinn_pde(pde_system, + discretization_05_gaussian; + draw_samples = 1000, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +# more iterations for above + strict BC +sol3_100_uniform_1000_bc = ahmc_bayesian_pinn_pde(pde_system, + discretization_uniform; + draw_samples = 1000, + bcstd = [0.1, 0.1], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_08_gaussian_1000_bc = ahmc_bayesian_pinn_pde(pde_system, + discretization_08_gaussian; + draw_samples = 1000, + bcstd = [0.1, 0.1], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_08_gaussian_1000_bc_hard = ahmc_bayesian_pinn_pde(pde_system, + discretization_08_gaussian; + draw_samples = 1000, + bcstd = [0.05, 0.05], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_05_gaussian_1000_bc = ahmc_bayesian_pinn_pde(pde_system, + discretization_05_gaussian; + draw_samples = 1000, + bcstd = [0.1, 0.1], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol3_100_08_gaussian_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_08_gaussian; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol3_100_05_gaussian_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_05_gaussian; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol4 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true +) + +sol4_0 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true +) + +sol5_00 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5_0 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +# 70 points in dataset +sol6 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +# SOL6_1 VS SOL6_L2 +sol6_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol6_2 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol6_2_L2 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol6_3 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol6_4 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol6_L2 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.2, 0.2], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol6_L2_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +sol6_L2_2 = ahmc_bayesian_pinn_pde(pde_system, + discretization1; + draw_samples = 700, + bcstd = [0.05, 0.05], + phystd = [0.2, 0.2], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 2), + Normal(2, 2), + Normal(2, 2) + ], progress = true) + +# 50 datapoint 0-5 sol5 vs sol4 +# julia> sol4.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.549 ± 0.0058 +# 0.71 ± 0.0042 +# 0.408 ± 0.0063 +# 0.355 ± 0.0015 + +# julia> sol5.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.604 ± 0.0052 +# 0.702 ± 0.0034 +# 0.346 ± 0.0037 +# 0.335 ± 0.0013 + +# 100 datapoint 0-5 sol5_2 vs sol3 +# julia> sol3.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.598 ± 0.0037 +# 0.711 ± 0.0027 +# 0.399 ± 0.0032 +# 0.333 ± 0.0011 + +# julia> sol5_2.estimated_de_params +# 4-element Vector{Particles{Float64, 234}}: +# 0.604 ± 0.0035 +# 0.686 ± 0.0026 +# 0.395 ± 0.0029 +# 0.328 ± 0.00095 + +# timespan for full dataset (0-8) +sol6 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], progress = true) + +sol5_3 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5_4 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], Dict_differentials = Dict_differentials, progress = true +) + +sol5_5 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], progress = true +) + +sol7 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(1, 2), + Normal(1, 1), + Normal(1, 2), + Normal(1, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol5_5_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], progress = true +) + +sol7_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.05, 0.05], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol7_2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.1, 0.1], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol7_3 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.2, 0.2], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +sol7_4 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 700, + bcstd = [0.1, 0.1], + phystd = [0.1, 0.1], l2std = [0.1, 0.1], + priorsNNw = (0.0, 5.0), + phystdnew = [0.3, 0.3], + saveats = [1 / 50.0], + param = [ + Normal(2, 2), + Normal(2, 1), + Normal(2, 2), + Normal(2, 1) + ], Dict_differentials = Dict_differentials, progress = true) + +using Plots, StatsPlots +plotly() +plot(time, u[1, :]) +plot!(time, u[2, :]) +scatter!(time, u_noisy[1, :]) +scatter!(time, u_noisy[2, :]) +scatter!(discretization.dataset[1][1][:, 2], discretization.dataset[1][1][:, 1]) +scatter!(discretization.dataset[1][2][:, 2], discretization.dataset[1][2][:, 1]) + +scatter!(discretization1.dataset[1][1][:, 2], + discretization1.dataset[1][1][:, 1], legend = nothing) +scatter!(discretization1.dataset[1][2][:, 2], discretization1.dataset[1][2][:, 1]) + +# plot28(sol4 seems better vs sol3 plots, params seems similar) +plot!(sol3.timepoints[1]', sol3.ensemblesol[1]) +plot!(sol3.timepoints[2]', sol3.ensemblesol[2]) +plot!(sol3_0.timepoints[1]', sol3_0.ensemblesol[1]) +plot!(sol3_0.timepoints[2]', sol3_0.ensemblesol[2]) + +plot!(sol4.timepoints[1]', sol4.ensemblesol[1]) +plot!(sol4.timepoints[2]', sol4.ensemblesol[2]) +plot!(sol4_0.timepoints[1]', sol4_0.ensemblesol[1]) +plot!(sol4_0.timepoints[2]', sol4_0.ensemblesol[2]) + +plot!(sol4_2.timepoints[1]', sol4_2.ensemblesol[1], legend = nothing) +plot!(sol4_2.timepoints[2]', sol4_2.ensemblesol[2]) +plot!(sol5_2.timepoints[1]', sol5_2.ensemblesol[1], legend = nothing) +plot!(sol5_2.timepoints[2]', sol5_2.ensemblesol[2]) + +plot!(sol4_3.timepoints[1]', sol4_3.ensemblesol[1], legend = nothing) +plot!(sol4_3.timepoints[2]', sol4_3.ensemblesol[2]) +plot!(sol5_3.timepoints[1]', sol5_3.ensemblesol[1]) +plot!(sol5_3.timepoints[2]', sol5_3.ensemblesol[2]) +plot!(sol5_4.timepoints[1]', sol5_4.ensemblesol[1], legend = nothing) +plot!(sol5_4.timepoints[2]', sol5_4.ensemblesol[2]) + +# plot 36 sol4 vs sol5(params sol4 better, but plots sol5 "looks" better),plot 44(sol5 better than sol6 overall) +plot!(sol5.timepoints[1]', sol5.ensemblesol[1], legend = nothing) +plot!(sol5.timepoints[2]', sol5.ensemblesol[2]) + +plot!(sol5_0.timepoints[1]', sol5_0.ensemblesol[1], legend = nothing) +plot!(sol5_0.timepoints[2]', sol5_0.ensemblesol[2]) + +plot!(sol5_00.timepoints[1]', sol5_00.ensemblesol[1], legend = nothing) +plot!(sol5_00.timepoints[2]', sol5_00.ensemblesol[2]) + +plot!(sol6.timepoints[1]', sol6.ensemblesol[1]) +plot!(sol6.timepoints[2]', sol6.ensemblesol[2]) +plot!(sol6_L2.timepoints[1]', sol6_L2.ensemblesol[1]) +plot!(sol6_L2.timepoints[2]', sol6_L2.ensemblesol[2]) + +plot!(sol6_L2_1.timepoints[1]', sol6_L2_1.ensemblesol[1]) +plot!(sol6_L2_1.timepoints[2]', sol6_L2_1.ensemblesol[2]) + +plot!(sol6_L2_2.timepoints[1]', sol6_L2_2.ensemblesol[1]) +plot!(sol6_L2_2.timepoints[2]', sol6_L2_2.ensemblesol[2]) + +plot!(sol6_1.timepoints[1]', sol6_1.ensemblesol[1]) +plot!(sol6_1.timepoints[2]', sol6_1.ensemblesol[2]) +plot!(sol6_2.timepoints[1]', sol6_2.ensemblesol[1]) +plot!(sol6_2.timepoints[2]', sol6_2.ensemblesol[2], legend = nothing) +plot!(sol6_2_L2.timepoints[1]', sol6_2_L2.ensemblesol[1]) +plot!(sol6_2_L2.timepoints[2]', sol6_2_L2.ensemblesol[2], legend = nothing) + +# plot52 sol7 vs sol5(sol5 overall better plots, params?) +plot!(sol7.timepoints[1]', sol7.ensemblesol[1]) +plot!(sol7.timepoints[2]', sol7.ensemblesol[2]) + +# sol8,sol8_2,sol9,sol9_2 bad +plot!(sol8.timepoints[1]', sol8.ensemblesol[1]) +plot!(sol8.timepoints[2]', sol8.ensemblesol[2]) +plot!(sol8_2.timepoints[1]', sol8_2.ensemblesol[1]) +plot!(sol8_2.timepoints[2]', sol8_2.ensemblesol[2]) + +plot!(sol9.timepoints[1]', sol9.ensemblesol[1]) +plot!(sol9.timepoints[2]', sol9.ensemblesol[2]) +plot!(sol9_2.timepoints[1]', sol9_2.ensemblesol[1]) +plot!(sol9_2.timepoints[2]', sol9_2.ensemblesol[2]) + +plot!(sol5_5.timepoints[1]', sol5_5.ensemblesol[1]) +plot!(sol5_5.timepoints[2]', sol5_5.ensemblesol[2], legend = nothing) + +plot!(sol5_5_1.timepoints[1]', sol5_5_1.ensemblesol[1]) +plot!(sol5_5_1.timepoints[2]', sol5_5_1.ensemblesol[2], legend = nothing) +plot!(sol7_1.timepoints[1]', sol7_1.ensemblesol[1]) +plot!(sol7_1.timepoints[2]', sol7_1.ensemblesol[2]) + +plot!(sol7_4.timepoints[1]', sol7_4.ensemblesol[1]) +plot!(sol7_4.timepoints[2]', sol7_4.ensemblesol[2]) + +plot!(sol5_2_1.timepoints[1]', sol5_2_1.ensemblesol[1], legend = nothing) +plot!(sol5_2_1.timepoints[2]', sol5_2_1.ensemblesol[2]) +plot!(sol5_2_2.timepoints[1]', sol5_2_2.ensemblesol[1], legend = nothing) +plot!(sol5_2_2.timepoints[2]', sol5_2_2.ensemblesol[2]) + +plot!(sol5_0.timepoints[1]', sol5_0.ensemblesol[1]) +plot!(sol5_0.timepoints[2]', sol5_0.ensemblesol[2], legend = nothing) + +plot!(sol5_00.timepoints[1]', sol5_00.ensemblesol[1], legend = nothing) +plot!(sol5_00.timepoints[2]', sol5_00.ensemblesol[2]) + +plot!(sol3_0.timepoints[1]', sol3_0.ensemblesol[1]) +plot!(sol3_0.timepoints[2]', sol3_0.ensemblesol[2], legend = nothing) +plot!(sol4_0.timepoints[1]', sol4_0.ensemblesol[1]) +plot!(sol4_0.timepoints[2]', sol4_0.ensemblesol[2], legend = nothing) + +plot!(sol3_100_05_gaussian.timepoints[1]', sol3_100_05_gaussian.ensemblesol[1]) +plot!(sol3_100_05_gaussian.timepoints[2]', + sol3_100_05_gaussian.ensemblesol[2], legend = nothing) + +plot!(sol3_100_05_gaussian_new.timepoints[1]', sol3_100_05_gaussian_new.ensemblesol[1]) +plot!(sol3_100_05_gaussian_new.timepoints[2]', sol3_100_05_gaussian_new.ensemblesol[2]) + +plot!(sol3_100_08_gaussian.timepoints[1]', sol3_100_08_gaussian.ensemblesol[1]) +plot!(sol3_100_08_gaussian.timepoints[2]', sol3_100_08_gaussian.ensemblesol[2]) + +plot!(sol3_100_08_gaussian_new.timepoints[1]', sol3_100_08_gaussian_new.ensemblesol[1]) +plot!(sol3_100_08_gaussian_new.timepoints[2]', + sol3_100_08_gaussian_new.ensemblesol[2], legend = nothing) + +plot!(sol3_100_uniform.timepoints[1]', sol3_100_uniform.ensemblesol[1]) +plot!(sol3_100_uniform.timepoints[2]', sol3_100_uniform.ensemblesol[2]) + +plot!(sol3_100_08_gaussian_1000.timepoints[1]', sol3_100_08_gaussian_1000.ensemblesol[1]) +plot!(sol3_100_08_gaussian_1000.timepoints[2]', sol3_100_08_gaussian_1000.ensemblesol[2]) + +plot!(sol3_100_05_gaussian_1000.timepoints[1]', sol3_100_05_gaussian_1000.ensemblesol[1]) +plot!(sol3_100_05_gaussian_1000.timepoints[2]', sol3_100_05_gaussian_1000.ensemblesol[2]) + +plot!(sol3_100_uniform_1000.timepoints[1]', sol3_100_uniform_1000.ensemblesol[1]) +plot!(sol3_100_uniform_1000.timepoints[2]', sol3_100_uniform_1000.ensemblesol[2]) + +plot!(sol3_100_08_gaussian_1000_bc.timepoints[1]', + sol3_100_08_gaussian_1000_bc.ensemblesol[1]) +plot!(sol3_100_08_gaussian_1000_bc.timepoints[2]', + sol3_100_08_gaussian_1000_bc.ensemblesol[2]) + +# test with lower number of points +# consider full range dataset case +# combination of all above + +# run 1 100 iters +sol5.estimated_de_params +sol6.estimated_de_params + +# run 2 200 iters +sol5.estimated_de_params +sol6.estimated_de_params + +# run 2 200 iters +sol3.estimated_de_params +sol4.estimated_de_params + +# p = [2/3, 2/3, 1/3, 1/3] +sol3.estimated_de_params +sol4.estimated_de_params + +@parameters t, p +@variables u(..) + +Dt = Differential(t) +eqs = Dt(u(t)) - cos(p * t) ~ 0 +bcs = [u(0) ~ 0.0] +domains = [t ∈ Interval(0.0, 2.0)] + +chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) +initl, st = Lux.setup(Random.default_rng(), chainl) + +@named pde_system = PDESystem(eqs, + bcs, + domains, + [t], + [u(t)], + [p], + defaults = Dict([p => 4.0])) + +analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) +timepoints = collect(0.0:(1 / 100.0):2.0) +u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] +u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) +dataset = [hcat(u1, timepoints)] + +discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, + dataset = [dataset, nothing]) + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.01], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(4.0, 2)], progress = true) + +param = 2 * π +ts = vec(sol1.timepoints[1]) +u_real = [analytic_sol_func1(0.0, t) for t in ts] +u_predict = pmean(sol1.ensemblesol[1]) + +@test u_predict≈u_real atol=0.1 +@test mean(u_predict .- u_real) < 0.01 +@test sol1.estimated_de_params[1]≈param atol=0.1 +sol1.estimated_de_params[1] + +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 1500, + bcstd = [0.05], + phystd = [0.01], l2std = [0.02], phystdnew = [0.02], + priorsNNw = (0.0, 1.0), + saveats = [1 / 50.0], + param = [LogNormal(4.0, 2)], + Dict_differentials = Dict_differentials, + progress = true) + +param = 2 * π +ts_2 = vec(sol2.timepoints[1]) +u_real_2 = [analytic_sol_func1(0.0, t) for t in ts] +u_predict_2 = pmean(sol2.ensemblesol[1]) + +@test u_predict_2≈u_real_2 atol=0.1 +@test mean(u_predict_2 .- u_real_2) < 0.01 +@test sol2.estimated_de_params[1]≈param atol=0.1 +sol2.estimated_de_params[1] + +plot(ts_2, u_predict_2) +plot!(ts_2, u_real_2) + +@parameters t, σ_ +@variables x(..), y(..), z(..) +Dt = Differential(t) +eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), + Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), + Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] + +bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] +domains = [t ∈ Interval(0.0, 1.0)] + +input_ = length(domains) +n = 7 +chain = [ + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)), + Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), + Lux.Dense(n, 1)) +] + +#Generate Data +function lorenz!(du, u, p, t) + du[1] = 10.0 * (u[2] - u[1]) + du[2] = u[1] * (28.0 - u[3]) - u[2] + du[3] = u[1] * u[2] - (8 / 3) * u[3] +end + +u0 = [1.0; 0.0; 0.0] +tspan = (0.0, 1.0) +prob = ODEProblem(lorenz!, u0, tspan) +sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) +ts = sol.t +us = hcat(sol.u...) +us = us .+ ((0.05 .* randn(size(us))) .* us) +ts_ = hcat(sol(ts).t...)[1, :] +dataset = [hcat(us[i, :], ts_) for i in 1:3] + +discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, + dataset = [dataset, nothing]) + +@named pde_system = PDESystem(eqs, bcs, domains, + [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 100, + bcstd = [0.3, 0.3, 0.3], + phystd = [0.1, 0.1, 0.1], + l2std = [1, 1, 1], + priorsNNw = (0.0, 1.0), + saveats = [0.01], + param = [Normal(14.0, 2)], progress = true) + +idealp = 10.0 +p_ = sol1.estimated_de_params[1] +@test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] +# @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] + +@parameters x y +@variables u(..) +Dxx = Differential(x)^2 +Dyy = Differential(y)^2 + +# 2D PDE +eq = Dxx(u(x, y)) + Dyy(u(x, y)) ~ -sin(pi * x) * sin(pi * y) + +# Boundary conditions +bcs = [u(0, y) ~ 0.0, u(1, y) ~ 0.0, + u(x, 0) ~ 0.0, u(x, 1) ~ 0.0] + +# Space and time domains +domains = [x ∈ Interval(0.0, 1.0), + y ∈ Interval(0.0, 1.0)] + +# Neural network +dim = 2 # number of dimensions +chain = Lux.Chain(Lux.Dense(dim, 9, Lux.σ), Lux.Dense(9, 9, Lux.σ), Lux.Dense(9, 1)) + +# Discretization +dx = 0.04 +discretization = BayesianPINN([chain], GridTraining(dx), dataset = [[dataset], nothing]) + +@named pde_system = PDESystem(eq, bcs, domains, [x, y], [u(x, y)]) + +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 5, + bcstd = [0.01, 0.01, 0.01, 0.01], + phystd = [0.005], + priorsNNw = (0.0, 2.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +xs = sol1.timepoints[1] +sol1.ensemblesol[1] +analytic_sol_func(x, y) = (sin(pi * x) * sin(pi * y)) / (2pi^2) + +dataset = hcat(u_real, xs') +u_predict = pmean(sol1.ensemblesol[1]) +u_real = [analytic_sol_func(xs[:, i][1], xs[:, i][2]) for i in 1:length(xs[1, :])] +@test u_predict≈u_real atol=0.8 + +# KS EQUATION +using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC +import ModelingToolkit: Interval, infimum, supremum, Distributions +using Plots, MonteCarloMeasurements, StatsPlots +# plotly() + +@parameters x, t, α +@variables u(..) +Dt = Differential(t) +Dx = Differential(x) +Dx2 = Differential(x)^2 +Dx3 = Differential(x)^3 +Dx4 = Differential(x)^4 + +# α = 1 +β = 4 +γ = 1 +eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 + +u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 + +bcs = [u(x, 0) ~ u_analytic(x, 0), + u(-10, t) ~ u_analytic(-10, t), + u(10, t) ~ u_analytic(10, t), + Dx(u(-10, t)) ~ du(-10, t), + Dx(u(10, t)) ~ du(10, t)] + +# Space and time domains +domains = [x ∈ Interval(-10.0, 10.0), + t ∈ Interval(0.0, 1.0)] + +# Discretization +dx = 0.4; +dt = 0.2; + +# Function to compute analytical solution at a specific point (x, t) +function u_analytic_point(x, t) + z = -x / 2 + t + return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 +end + +# Function to generate the dataset matrix +function generate_dataset_matrix(domains, dx, dt, xlim, tlim) + x_values = xlim[1]:dx:xlim[2] + t_values = tlim[1]:dt:tlim[2] + + dataset = [] + + for t in t_values + for x in x_values + u_value = u_analytic_point(x, t) + push!(dataset, [u_value, x, t]) + end + end + + return vcat([data' for data in dataset]...) +end + +datasetpde = [generate_dataset_matrix(domains, dx, dt, [-10, 10], [0.0, 1.0])] +datasetpde_new = [generate_dataset_matrix(domains, dx, dt, [-10, 0], [0.0, 1.0])] + +# noise to dataset +noisydataset = deepcopy(datasetpde) +noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ (randn(size(noisydataset[1][:, 1])) .* 0.8) + +noisydataset_new = deepcopy(datasetpde_new) +noisydataset_new[1][:, 1] = noisydataset_new[1][:, 1] .+ + (randn(size(noisydataset_new[1][:, 1])) .* 0.8) + +plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") +scatter!(noisydataset[1][:, 2], noisydataset[1][:, 1]) + +plot(datasetpde[1][:, 2], datasetpde[1][:, 3], datasetpde[1][:, 1], + title = "Dataset from Analytical Solution") +scatter!(noisydataset[1][:, 2], noisydataset[1][:, 3], noisydataset[1][:, 1]) + +plot(datasetpde_new[1][:, 2], datasetpde_new[1][:, 1], + title = "Dataset from Analytical Solution") +scatter!(noisydataset_new[1][:, 2], noisydataset_new[1][:, 1]) + +plot(datasetpde_new[1][:, 2], datasetpde_new[1][:, 3], + datasetpde_new[1][:, 1], title = "Dataset from Analytical Solution") +scatter!(noisydataset_new[1][:, 2], noisydataset_new[1][:, 3], noisydataset_new[1][:, 1]) + +noise_std = 1.4 +original_data = datasetpde[1][:, 1] +original_std = std(original_data) +ratio = noise_std / original_std + +# Neural network +chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), + Lux.Dense(8, 8, Lux.tanh), + Lux.Dense(8, 1)) + +discretization = NeuralPDE.BayesianPINN([chain], + GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) + +discretization_new = NeuralPDE.BayesianPINN([chain], + GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset_new, nothing]) + +@named pde_system = PDESystem(eq, + bcs, + domains, + [x, t], + [u(x, t)], + [α], + defaults = Dict([α => 2.0])) + +eqs = pde_system.eqs +Dict_differentials = Dict() +exps = toexpr.(eqs) +nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + +sol1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.5], + phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol1_1 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 90, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.7], + phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol2 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +sol3 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 100, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.7], l2std = [0.15], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 3.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +sol4 = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1_1.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_1.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol3.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol3.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol4.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol4.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +sol0_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], + phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +julia > sol5_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 170, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +sol1_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], + phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol1_1_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.5], + phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol1_2_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 150, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.1], + phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 3.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol1_3_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 150, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], + phystd = [0.3], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 3.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + +sol2_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 140, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +sol3_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 140, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +sol4_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 160, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +sol5_new = ahmc_bayesian_pinn_pde(pde_system, + discretization_new; + draw_samples = 170, Kernel = AdvancedHMC.NUTS(0.8), + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol0_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol0_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol1_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1_1_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol1_1_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1_2_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol1_2_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol1_3_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol1_3_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol2_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol2_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol3_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol3_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol4_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol4_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol5_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol5_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + +p1 = plot(xs, u_predict, title = "predict") +p2 = plot(xs, u_real, title = "analytic") +p3 = plot(xs, diff_u, title = "error") +plot(p1, p2, p3) + +phi = discretization_new.phi[1] +xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + +u_predict = [[first(pmean(phi([x, t], sol5_new.estimated_nn_params[1]))) for x in xs] + for t in ts] +u_real = [[u_analytic(x, t) for x in xs] for t in ts] +diff_u = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol5_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] +p1 = plot(ts, xs, u_predict, title = "predict") +p2 = plot(ts, xs, u_real, title = "analytic") +p3 = plot(ts, xs, diff_u, title = "error") +plot(p1, p2, p3) + +# MCMC chain analysis +plot(sol1.original.mcmc_chain) +plot(sol2.original.mcmc_chain) + +plot(sol0_new.original.mcmc_chain) +plot(sol2_new.original.mcmc_chain) + +plot(sol1.original.mcmc_chain) +meanplot(sol1.original.mcmc_chain) +autocorplot(sol1.original.mcmc_chain) +traceplot(sol1.original.mcmc_chain) + +plot(sol2.original.mcmc_chain) +meanplot(sol2.original.mcmc_chain) +autocorplot(sol2.original.mcmc_chain) +traceplot(sol2.original.mcmc_chain) + +plot(sol0_new.original.mcmc_chain) +meanplot(sol0_new.original.mcmc_chain) +autocorplot(sol0_new.original.mcmc_chain) + +plot(sol2_new.original.mcmc_chain) +meanplot(sol2_new.original.mcmc_chain) +autocorplot(sol2_new.original.mcmc_chain) + +plot(sol3_new.original.mcmc_chain) +meanplot(sol3_new.original.mcmc_chain) +autocorplot(sol3_new.original.mcmc_chain) \ No newline at end of file diff --git a/test/bpinnexperimental.jl b/test/bpinnexperimental.jl deleted file mode 100644 index a8a389ad44..0000000000 --- a/test/bpinnexperimental.jl +++ /dev/null @@ -1,140 +0,0 @@ -using Test, MCMCChains -using ForwardDiff, Distributions, OrdinaryDiffEq -using Flux, OptimizationOptimisers, AdvancedHMC, Lux -using Statistics, Random, Functors, ComponentArrays -using NeuralPDE, MonteCarloMeasurements - -Random.seed!(110) - -using NeuralPDE, Lux, Plots, OrdinaryDiffEq, Distributions, Random - -function lotka_volterra(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end - -# initial-value problem. -u0 = [1.0, 1.0] -p = [1.5, 1.0, 3.0, 1.0] -tspan = (0.0, 4.0) -prob = ODEProblem(lotka_volterra, u0, tspan, p) - -# Solve using OrdinaryDiffEq.jl solver -dt = 0.2 -solution = solve(prob, Tsit5(); saveat = dt) - -times = solution.t -u = hcat(solution.u...) -x = u[1, :] + (u[1, :]) .* (0.3 .* randn(length(u[1, :]))) -y = u[2, :] + (u[2, :]) .* (0.3 .* randn(length(u[2, :]))) -dataset = [x, y, times] - -plot(times, x, label = "noisy x") -plot!(times, y, label = "noisy y") -plot!(solution, labels = ["x" "y"]) - -chain = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), - Lux.Dense(6, 2)) - -alg1 = BNNODE(chain; - dataset = dataset, - draw_samples = 1000, - l2std = [0.1, 0.1], - phystd = [0.1, 0.1], - priorsNNw = (0.0, 3.0), - param = [ - Normal(1, 2), - Normal(2, 2), - Normal(2, 2), - Normal(0, 2)], progress = true) - -alg2 = BNNODE(chain; - dataset = dataset, - draw_samples = 1000, - l2std = [0.1, 0.1], - phystd = [0.1, 0.1], - priorsNNw = (0.0, 3.0), - param = [ - Normal(1, 2), - Normal(2, 2), - Normal(2, 2), - Normal(0, 2)], estim_collocate = true, progress = true) - -@time sol_pestim1 = solve(prob, alg1; saveat = dt) -@time sol_pestim2 = solve(prob, alg2; saveat = dt) -plot(times, sol_pestim1.ensemblesol[1], label = "estimated x1") -plot!(times, sol_pestim2.ensemblesol[1], label = "estimated x2") -plot!(times, sol_pestim1.ensemblesol[2], label = "estimated y1") -plot!(times, sol_pestim2.ensemblesol[2], label = "estimated y2") - -# comparing it with the original solution -plot!(solution, labels = ["true x" "true y"]) - -@show sol_pestim1.estimated_de_params -@show sol_pestim2.estimated_de_params - -function fitz(u, p, t) - v, w = u[1], u[2] - a, b, τinv, l = p[1], p[2], p[3], p[4] - - dv = v - 0.33 * v^3 - w + l - dw = τinv * (v + a - b * w) - - return [dv, dw] -end - -prob_ode_fitzhughnagumo = ODEProblem( - fitz, [1.0, 1.0], (0.0, 10.0), [0.7, 0.8, 1 / 12.5, 0.5]) -dt = 0.5 -sol = solve(prob_ode_fitzhughnagumo, Tsit5(), saveat = dt) - -sig = 0.20 -data = Array(sol) -dataset = [data[1, :] .+ (sig .* rand(length(sol.t))), - data[2, :] .+ (sig .* rand(length(sol.t))), sol.t] -priors = [Normal(0.5, 1.0), Normal(0.5, 1.0), Normal(0.0, 0.5), Normal(0.5, 1.0)] - -plot(sol.t, dataset[1], label = "noisy x") -plot!(sol.t, dataset[2], label = "noisy y") -plot!(sol, labels = ["x" "y"]) - -chain = Lux.Chain(Lux.Dense(1, 10, tanh), Lux.Dense(10, 10, tanh), - Lux.Dense(10, 2)) - -Adaptorkwargs = (Adaptor = AdvancedHMC.StanHMCAdaptor, - Metric = AdvancedHMC.DiagEuclideanMetric, targetacceptancerate = 0.8) -alg1 = BNNODE(chain; -dataset = dataset, -draw_samples = 1000, -l2std = [0.1, 0.1], -phystd = [0.1, 0.1], -priorsNNw = (0.01, 3.0), -Adaptorkwargs = Adaptorkwargs, -param = priors, progress = true) - -alg2 = BNNODE(chain; - dataset = dataset, - draw_samples = 1000, - l2std = [0.1, 0.1], - phystd = [0.1, 0.1], - priorsNNw = (0.01, 3.0), - Adaptorkwargs = Adaptorkwargs, - param = priors, estim_collocate = true, progress = true) - -@time sol_pestim3 = solve(prob_ode_fitzhughnagumo, alg1; saveat = dt) -@time sol_pestim4 = solve(prob_ode_fitzhughnagumo, alg2; saveat = dt) -plot!(sol.t, sol_pestim3.ensemblesol[1], label = "estimated x1") -plot!(sol.t, sol_pestim4.ensemblesol[1], label = "estimated x2") -plot!(sol.t, sol_pestim3.ensemblesol[2], label = "estimated y1") -plot!(sol.t, sol_pestim4.ensemblesol[2], label = "estimated y2") - -@show sol_pestim3.estimated_de_params -@show sol_pestim4.estimated_de_params From 94770d4a96fc9ee15deaa0ce8296adafce69aa08 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sun, 12 May 2024 11:13:18 +0530 Subject: [PATCH 45/53] need PDE exp file to be concise --- src/NeuralPDE.jl | 1 - test/BPINN_pde_experimental.jl | 1669 -------------------------------- 2 files changed, 1670 deletions(-) delete mode 100644 test/BPINN_pde_experimental.jl diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 920387340a..1122afc838 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -54,7 +54,6 @@ include("advancedHMC_MCMC.jl") include("BPINN_ode.jl") include("PDE_BPINN.jl") include("dgm.jl") -include("collocated_estim.jl") export NNODE, NNDAE, PhysicsInformedNN, discretize, diff --git a/test/BPINN_pde_experimental.jl b/test/BPINN_pde_experimental.jl deleted file mode 100644 index a8f4a0341e..0000000000 --- a/test/BPINN_pde_experimental.jl +++ /dev/null @@ -1,1669 +0,0 @@ -using Test, MCMCChains, Lux, ModelingToolkit -import ModelingToolkit: Interval, infimum, supremum -using ForwardDiff, Distributions, OrdinaryDiffEq -using AdvancedHMC, Statistics, Random, Functors -using NeuralPDE, MonteCarloMeasurements -using ComponentArrays, ModelingToolkit - -Random.seed!(100) - -# function required to use the new loss, creates a dicitonary of differntial operator terms -function recur_expression(exp, Dict_differentials) - for in_exp in exp.args - if !(in_exp isa Expr) - # skip +,== symbols, characters etc - continue - - elseif in_exp.args[1] isa ModelingToolkit.Differential - # first symbol of differential term - # Dict_differentials for masking differential terms - # and resubstituting differentials in equations after putting in interpolations - # temp = in_exp.args[end] - Dict_differentials[eval(in_exp)] = Symbolics.variable("diff_$(length(Dict_differentials) + 1)") - return - else - recur_expression(in_exp, Dict_differentials) - end - end -end - -# experiments are here -println("Example 3: Lotka Volterra with New parameter estimation") -@parameters t α β γ δ -@variables x(..) y(..) - -Dt = Differential(t) -eqs = [Dt(x(t)) * α ~ x(t) - β * x(t) * y(t), Dt(y(t)) * δ ~ x(t) * y(t) - y(t) * γ] -bcs = [x(0) ~ 1.0, y(0) ~ 1.0] -domains = [t ∈ Interval(0.0, 7.0)] - -# Define the parameters' values -# α, β, γ, δ = p - -# regular equations -# dx = (1.5 - y) * x # prey -# dy = (x - 3.0) * y # predator -# p = [1.5, 1.0, 3.0, 1.0] non transformed values - -# transformed equations -# dx*0.666 = (1 - 0.666 * y) * x # prey -# dy*1.0 = (x - 3.0) * y # predator -# p = [0.666, 0.666, 3.0, 1.0] transformed values (change is scale also ensured!) - -chainl = [ - Lux.Chain(Lux.Dense(1, 5, sin), Lux.Dense(5, 5, sin), Lux.Dense(5, 1)), - Lux.Chain(Lux.Dense(1, 5, sin), Lux.Dense(5, 5, sin), Lux.Dense(5, 1)) -] - -initl, st = Lux.setup(Random.default_rng(), chainl[1]) -initl1, st1 = Lux.setup(Random.default_rng(), chainl[2]) - -using NeuralPDE, Lux, OrdinaryDiffEq, Distributions, Random - -function lotka_volterra(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] -end -# initial-value problem. -u0 = [1.0, 1.0] -# p = [2/3, 2/3, 1/3.0, 1/3.0] -p = [1.5, 1.0, 3.0, 1.0] -tspan = (0.0, 7.0) -prob = ODEProblem(lotka_volterra, u0, tspan, p) -dt = 0.01 -solution = solve(prob, Tsit5(); saveat = dt) -solution1 = solve(prob, Tsit5(); saveat = 0.02) - -function calculate_errors(approx_sol, solution_points) - # Check vector lengths match - if length(approx_sol) != length(solution_points) - error("Vectors must have the same length") - end - - # Calculate errors - n = length(approx_sol) - errors = randn(n) - for i in 1:n - errors[i] = solution_points[i] - approx_sol[i] - end - - # Calculate RMSE - rmse = sqrt(mean(errors .^ 2)) - - # Calculate MAE - mae = mean(abs.(errors)) - - # Calculate maximum absolute error - max_error = maximum(abs.(errors)) - - # Return dictionary with errors - return Dict( - "RMSE" => rmse, - "MAE" => mae, - "Max Abs Error" => max_error - ) -end -u = hcat(solution1.u...) - -a1 = calculate_errors(pmean(sol6_1.ensemblesol[1]), u1[1, :]) -b1 = calculate_errors(pmean(sol6_1.ensemblesol[2]), u1[2, :]) - -a = calculate_errors(pmean(sol6_2.ensemblesol[1]), u[1, :]) -b = calculate_errors(pmean(sol6_2.ensemblesol[2]), u[2, :]) - -c = calculate_errors(pmean(sol6_L2_2.ensemblesol[1]), u[1, :]) -d = calculate_errors(pmean(sol6_L2_2.ensemblesol[2]), u[2, :]) - -e = calculate_errors(pmean(sol6_L2_1.ensemblesol[1]), u[1, :]) -f = calculate_errors(pmean(sol6_L2_1.ensemblesol[2]), u[2, :]) - -g = calculate_errors(pmean(sol6_L2.ensemblesol[1]), u[1, :]) -h = calculate_errors(pmean(sol6_L2.ensemblesol[2]), u[2, :]) - -# function moving_average_smoothing(data::Vector{T}, window_size::Int) where {T} -# smoothed_data = similar(data, T, length(data)) - -# for i in 1:length(data) -# start_idx = max(1, i - window_size) -# end_idx = min(length(data), i + window_size) -# smoothed_data[i] = mean(data[start_idx:end_idx]) -# end - -# return smoothed_data' -# end - -# Extract solution -time = solution.t -u = hcat(solution.u...) -time1 = solution.t -u_noisy = u .+ u .* (0.2 .* randn(size(u))) -u_noisy0 = u .+ (3.0 .* rand(size(u)[1], size(u)[2]) .- 1.5) -u_noisy1 = u .+ (0.8 .* randn(size(Array(solution)))) -u_noisy2 = u .+ (0.5 .* randn(size(Array(solution)))) - -plot(time, u[1, :]) -plot!(time, u[2, :]) -scatter!(time1, u_noisy0[1, :]) -scatter!(time1, u_noisy0[2, :]) -scatter!(discretization_08_gaussian.dataset[1][1][:, 2], - discretization_08_gaussian.dataset[1][1][:, 1]) -scatter!(discretization_08_gaussian.dataset[1][2][:, 2], - discretization_08_gaussian.dataset[1][2][:, 1]) - -scatter!(discretization_05_gaussian.dataset[1][1][:, 2], - discretization_05_gaussian.dataset[1][1][:, 1]) -scatter!(discretization_05_gaussian.dataset[1][2][:, 2], - discretization_05_gaussian.dataset[1][2][:, 1]) -# discretization_05_gaussian.dataset[1][1][:,2] -# window_size = 5 -# smoothed_datasets = [moving_average_smoothing(u1[i, :], window_size) -# for i in 1:length(solution.u[1])] -# u2 = vcat(smoothed_datasets[1], smoothed_datasets[2]) - -# Randomly select some points from the solution -num_points = 100 # Number of points to select -selected_indices = rand(1:size(u_noisy1, 2), num_points) -upoints = [u_noisy1[:, i] for i in selected_indices] -timepoints = [time[i] for i in selected_indices] -temp = hcat(upoints...) -dataset = [hcat(temp[i, :], timepoints) for i in 1:2] - -discretization_uniform = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, - dataset = [dataset, nothing]) -discretization_08_gaussian = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, - dataset = [dataset, nothing]) -discretization_05_gaussian = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, - dataset = [dataset, nothing]) - -discretization1 = BayesianPINN(chainl, GridTraining([0.01]), param_estim = true, - dataset = [dataset, nothing]) - -scatter!(discretization.dataset[1][1][:, 2], discretization.dataset[1][1][:, 1]) -scatter!(discretization.dataset[1][2][:, 2], discretization.dataset[1][2][:, 1]) - -sol = solve(prob, Tsit5(); saveat = 0.1) -odedata = Array(sol) + 0.8 * randn(size(Array(sol))) - -@named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [x(t), y(t)], - [α, β, γ, δ], - defaults = Dict([α => 2, β => 2, γ => 2, δ => 2])) - -# creating dictionary for masking equations -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol3 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_uniform = ahmc_bayesian_pinn_pde(pde_system, - discretization_uniform; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_08_gaussian = ahmc_bayesian_pinn_pde(pde_system, - discretization_08_gaussian; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_05_gaussian = ahmc_bayesian_pinn_pde(pde_system, - discretization_05_gaussian; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -# more iterations for above -sol3_100_uniform_1000 = ahmc_bayesian_pinn_pde(pde_system, - discretization_uniform; - draw_samples = 1000, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_08_gaussian_1000 = ahmc_bayesian_pinn_pde(pde_system, - discretization_08_gaussian; - draw_samples = 1000, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_05_gaussian_1000 = ahmc_bayesian_pinn_pde(pde_system, - discretization_05_gaussian; - draw_samples = 1000, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -# more iterations for above + strict BC -sol3_100_uniform_1000_bc = ahmc_bayesian_pinn_pde(pde_system, - discretization_uniform; - draw_samples = 1000, - bcstd = [0.1, 0.1], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_08_gaussian_1000_bc = ahmc_bayesian_pinn_pde(pde_system, - discretization_08_gaussian; - draw_samples = 1000, - bcstd = [0.1, 0.1], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_08_gaussian_1000_bc_hard = ahmc_bayesian_pinn_pde(pde_system, - discretization_08_gaussian; - draw_samples = 1000, - bcstd = [0.05, 0.05], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_05_gaussian_1000_bc = ahmc_bayesian_pinn_pde(pde_system, - discretization_05_gaussian; - draw_samples = 1000, - bcstd = [0.1, 0.1], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol3_100_08_gaussian_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_08_gaussian; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol3_100_05_gaussian_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_05_gaussian; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol4 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true -) - -sol4_0 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true -) - -sol5_00 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5_0 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -# 70 points in dataset -sol6 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -# SOL6_1 VS SOL6_L2 -sol6_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol6_2 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol6_2_L2 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol6_3 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol6_4 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol6_L2 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.2, 0.2], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol6_L2_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -sol6_L2_2 = ahmc_bayesian_pinn_pde(pde_system, - discretization1; - draw_samples = 700, - bcstd = [0.05, 0.05], - phystd = [0.2, 0.2], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 2), - Normal(2, 2), - Normal(2, 2) - ], progress = true) - -# 50 datapoint 0-5 sol5 vs sol4 -# julia> sol4.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.549 ± 0.0058 -# 0.71 ± 0.0042 -# 0.408 ± 0.0063 -# 0.355 ± 0.0015 - -# julia> sol5.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.604 ± 0.0052 -# 0.702 ± 0.0034 -# 0.346 ± 0.0037 -# 0.335 ± 0.0013 - -# 100 datapoint 0-5 sol5_2 vs sol3 -# julia> sol3.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.598 ± 0.0037 -# 0.711 ± 0.0027 -# 0.399 ± 0.0032 -# 0.333 ± 0.0011 - -# julia> sol5_2.estimated_de_params -# 4-element Vector{Particles{Float64, 234}}: -# 0.604 ± 0.0035 -# 0.686 ± 0.0026 -# 0.395 ± 0.0029 -# 0.328 ± 0.00095 - -# timespan for full dataset (0-8) -sol6 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], progress = true) - -sol5_3 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5_4 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], Dict_differentials = Dict_differentials, progress = true -) - -sol5_5 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], progress = true -) - -sol7 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(1, 2), - Normal(1, 1), - Normal(1, 2), - Normal(1, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol5_5_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], progress = true -) - -sol7_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.05, 0.05], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol7_2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.1, 0.1], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol7_3 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.2, 0.2], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -sol7_4 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 700, - bcstd = [0.1, 0.1], - phystd = [0.1, 0.1], l2std = [0.1, 0.1], - priorsNNw = (0.0, 5.0), - phystdnew = [0.3, 0.3], - saveats = [1 / 50.0], - param = [ - Normal(2, 2), - Normal(2, 1), - Normal(2, 2), - Normal(2, 1) - ], Dict_differentials = Dict_differentials, progress = true) - -using Plots, StatsPlots -plotly() -plot(time, u[1, :]) -plot!(time, u[2, :]) -scatter!(time, u_noisy[1, :]) -scatter!(time, u_noisy[2, :]) -scatter!(discretization.dataset[1][1][:, 2], discretization.dataset[1][1][:, 1]) -scatter!(discretization.dataset[1][2][:, 2], discretization.dataset[1][2][:, 1]) - -scatter!(discretization1.dataset[1][1][:, 2], - discretization1.dataset[1][1][:, 1], legend = nothing) -scatter!(discretization1.dataset[1][2][:, 2], discretization1.dataset[1][2][:, 1]) - -# plot28(sol4 seems better vs sol3 plots, params seems similar) -plot!(sol3.timepoints[1]', sol3.ensemblesol[1]) -plot!(sol3.timepoints[2]', sol3.ensemblesol[2]) -plot!(sol3_0.timepoints[1]', sol3_0.ensemblesol[1]) -plot!(sol3_0.timepoints[2]', sol3_0.ensemblesol[2]) - -plot!(sol4.timepoints[1]', sol4.ensemblesol[1]) -plot!(sol4.timepoints[2]', sol4.ensemblesol[2]) -plot!(sol4_0.timepoints[1]', sol4_0.ensemblesol[1]) -plot!(sol4_0.timepoints[2]', sol4_0.ensemblesol[2]) - -plot!(sol4_2.timepoints[1]', sol4_2.ensemblesol[1], legend = nothing) -plot!(sol4_2.timepoints[2]', sol4_2.ensemblesol[2]) -plot!(sol5_2.timepoints[1]', sol5_2.ensemblesol[1], legend = nothing) -plot!(sol5_2.timepoints[2]', sol5_2.ensemblesol[2]) - -plot!(sol4_3.timepoints[1]', sol4_3.ensemblesol[1], legend = nothing) -plot!(sol4_3.timepoints[2]', sol4_3.ensemblesol[2]) -plot!(sol5_3.timepoints[1]', sol5_3.ensemblesol[1]) -plot!(sol5_3.timepoints[2]', sol5_3.ensemblesol[2]) -plot!(sol5_4.timepoints[1]', sol5_4.ensemblesol[1], legend = nothing) -plot!(sol5_4.timepoints[2]', sol5_4.ensemblesol[2]) - -# plot 36 sol4 vs sol5(params sol4 better, but plots sol5 "looks" better),plot 44(sol5 better than sol6 overall) -plot!(sol5.timepoints[1]', sol5.ensemblesol[1], legend = nothing) -plot!(sol5.timepoints[2]', sol5.ensemblesol[2]) - -plot!(sol5_0.timepoints[1]', sol5_0.ensemblesol[1], legend = nothing) -plot!(sol5_0.timepoints[2]', sol5_0.ensemblesol[2]) - -plot!(sol5_00.timepoints[1]', sol5_00.ensemblesol[1], legend = nothing) -plot!(sol5_00.timepoints[2]', sol5_00.ensemblesol[2]) - -plot!(sol6.timepoints[1]', sol6.ensemblesol[1]) -plot!(sol6.timepoints[2]', sol6.ensemblesol[2]) -plot!(sol6_L2.timepoints[1]', sol6_L2.ensemblesol[1]) -plot!(sol6_L2.timepoints[2]', sol6_L2.ensemblesol[2]) - -plot!(sol6_L2_1.timepoints[1]', sol6_L2_1.ensemblesol[1]) -plot!(sol6_L2_1.timepoints[2]', sol6_L2_1.ensemblesol[2]) - -plot!(sol6_L2_2.timepoints[1]', sol6_L2_2.ensemblesol[1]) -plot!(sol6_L2_2.timepoints[2]', sol6_L2_2.ensemblesol[2]) - -plot!(sol6_1.timepoints[1]', sol6_1.ensemblesol[1]) -plot!(sol6_1.timepoints[2]', sol6_1.ensemblesol[2]) -plot!(sol6_2.timepoints[1]', sol6_2.ensemblesol[1]) -plot!(sol6_2.timepoints[2]', sol6_2.ensemblesol[2], legend = nothing) -plot!(sol6_2_L2.timepoints[1]', sol6_2_L2.ensemblesol[1]) -plot!(sol6_2_L2.timepoints[2]', sol6_2_L2.ensemblesol[2], legend = nothing) - -# plot52 sol7 vs sol5(sol5 overall better plots, params?) -plot!(sol7.timepoints[1]', sol7.ensemblesol[1]) -plot!(sol7.timepoints[2]', sol7.ensemblesol[2]) - -# sol8,sol8_2,sol9,sol9_2 bad -plot!(sol8.timepoints[1]', sol8.ensemblesol[1]) -plot!(sol8.timepoints[2]', sol8.ensemblesol[2]) -plot!(sol8_2.timepoints[1]', sol8_2.ensemblesol[1]) -plot!(sol8_2.timepoints[2]', sol8_2.ensemblesol[2]) - -plot!(sol9.timepoints[1]', sol9.ensemblesol[1]) -plot!(sol9.timepoints[2]', sol9.ensemblesol[2]) -plot!(sol9_2.timepoints[1]', sol9_2.ensemblesol[1]) -plot!(sol9_2.timepoints[2]', sol9_2.ensemblesol[2]) - -plot!(sol5_5.timepoints[1]', sol5_5.ensemblesol[1]) -plot!(sol5_5.timepoints[2]', sol5_5.ensemblesol[2], legend = nothing) - -plot!(sol5_5_1.timepoints[1]', sol5_5_1.ensemblesol[1]) -plot!(sol5_5_1.timepoints[2]', sol5_5_1.ensemblesol[2], legend = nothing) -plot!(sol7_1.timepoints[1]', sol7_1.ensemblesol[1]) -plot!(sol7_1.timepoints[2]', sol7_1.ensemblesol[2]) - -plot!(sol7_4.timepoints[1]', sol7_4.ensemblesol[1]) -plot!(sol7_4.timepoints[2]', sol7_4.ensemblesol[2]) - -plot!(sol5_2_1.timepoints[1]', sol5_2_1.ensemblesol[1], legend = nothing) -plot!(sol5_2_1.timepoints[2]', sol5_2_1.ensemblesol[2]) -plot!(sol5_2_2.timepoints[1]', sol5_2_2.ensemblesol[1], legend = nothing) -plot!(sol5_2_2.timepoints[2]', sol5_2_2.ensemblesol[2]) - -plot!(sol5_0.timepoints[1]', sol5_0.ensemblesol[1]) -plot!(sol5_0.timepoints[2]', sol5_0.ensemblesol[2], legend = nothing) - -plot!(sol5_00.timepoints[1]', sol5_00.ensemblesol[1], legend = nothing) -plot!(sol5_00.timepoints[2]', sol5_00.ensemblesol[2]) - -plot!(sol3_0.timepoints[1]', sol3_0.ensemblesol[1]) -plot!(sol3_0.timepoints[2]', sol3_0.ensemblesol[2], legend = nothing) -plot!(sol4_0.timepoints[1]', sol4_0.ensemblesol[1]) -plot!(sol4_0.timepoints[2]', sol4_0.ensemblesol[2], legend = nothing) - -plot!(sol3_100_05_gaussian.timepoints[1]', sol3_100_05_gaussian.ensemblesol[1]) -plot!(sol3_100_05_gaussian.timepoints[2]', - sol3_100_05_gaussian.ensemblesol[2], legend = nothing) - -plot!(sol3_100_05_gaussian_new.timepoints[1]', sol3_100_05_gaussian_new.ensemblesol[1]) -plot!(sol3_100_05_gaussian_new.timepoints[2]', sol3_100_05_gaussian_new.ensemblesol[2]) - -plot!(sol3_100_08_gaussian.timepoints[1]', sol3_100_08_gaussian.ensemblesol[1]) -plot!(sol3_100_08_gaussian.timepoints[2]', sol3_100_08_gaussian.ensemblesol[2]) - -plot!(sol3_100_08_gaussian_new.timepoints[1]', sol3_100_08_gaussian_new.ensemblesol[1]) -plot!(sol3_100_08_gaussian_new.timepoints[2]', - sol3_100_08_gaussian_new.ensemblesol[2], legend = nothing) - -plot!(sol3_100_uniform.timepoints[1]', sol3_100_uniform.ensemblesol[1]) -plot!(sol3_100_uniform.timepoints[2]', sol3_100_uniform.ensemblesol[2]) - -plot!(sol3_100_08_gaussian_1000.timepoints[1]', sol3_100_08_gaussian_1000.ensemblesol[1]) -plot!(sol3_100_08_gaussian_1000.timepoints[2]', sol3_100_08_gaussian_1000.ensemblesol[2]) - -plot!(sol3_100_05_gaussian_1000.timepoints[1]', sol3_100_05_gaussian_1000.ensemblesol[1]) -plot!(sol3_100_05_gaussian_1000.timepoints[2]', sol3_100_05_gaussian_1000.ensemblesol[2]) - -plot!(sol3_100_uniform_1000.timepoints[1]', sol3_100_uniform_1000.ensemblesol[1]) -plot!(sol3_100_uniform_1000.timepoints[2]', sol3_100_uniform_1000.ensemblesol[2]) - -plot!(sol3_100_08_gaussian_1000_bc.timepoints[1]', - sol3_100_08_gaussian_1000_bc.ensemblesol[1]) -plot!(sol3_100_08_gaussian_1000_bc.timepoints[2]', - sol3_100_08_gaussian_1000_bc.ensemblesol[2]) - -# test with lower number of points -# consider full range dataset case -# combination of all above - -# run 1 100 iters -sol5.estimated_de_params -sol6.estimated_de_params - -# run 2 200 iters -sol5.estimated_de_params -sol6.estimated_de_params - -# run 2 200 iters -sol3.estimated_de_params -sol4.estimated_de_params - -# p = [2/3, 2/3, 1/3, 1/3] -sol3.estimated_de_params -sol4.estimated_de_params - -@parameters t, p -@variables u(..) - -Dt = Differential(t) -eqs = Dt(u(t)) - cos(p * t) ~ 0 -bcs = [u(0) ~ 0.0] -domains = [t ∈ Interval(0.0, 2.0)] - -chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) -initl, st = Lux.setup(Random.default_rng(), chainl) - -@named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [u(t)], - [p], - defaults = Dict([p => 4.0])) - -analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) -timepoints = collect(0.0:(1 / 100.0):2.0) -u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] -u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) -dataset = [hcat(u1, timepoints)] - -discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, - dataset = [dataset, nothing]) - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(4.0, 2)], progress = true) - -param = 2 * π -ts = vec(sol1.timepoints[1]) -u_real = [analytic_sol_func1(0.0, t) for t in ts] -u_predict = pmean(sol1.ensemblesol[1]) - -@test u_predict≈u_real atol=0.1 -@test mean(u_predict .- u_real) < 0.01 -@test sol1.estimated_de_params[1]≈param atol=0.1 -sol1.estimated_de_params[1] - -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.02], phystdnew = [0.02], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(4.0, 2)], - Dict_differentials = Dict_differentials, - progress = true) - -param = 2 * π -ts_2 = vec(sol2.timepoints[1]) -u_real_2 = [analytic_sol_func1(0.0, t) for t in ts] -u_predict_2 = pmean(sol2.ensemblesol[1]) - -@test u_predict_2≈u_real_2 atol=0.1 -@test mean(u_predict_2 .- u_real_2) < 0.01 -@test sol2.estimated_de_params[1]≈param atol=0.1 -sol2.estimated_de_params[1] - -plot(ts_2, u_predict_2) -plot!(ts_2, u_real_2) - -@parameters t, σ_ -@variables x(..), y(..), z(..) -Dt = Differential(t) -eqs = [Dt(x(t)) ~ σ_ * (y(t) - x(t)), - Dt(y(t)) ~ x(t) * (28.0 - z(t)) - y(t), - Dt(z(t)) ~ x(t) * y(t) - 8 / 3 * z(t)] - -bcs = [x(0) ~ 1.0, y(0) ~ 0.0, z(0) ~ 0.0] -domains = [t ∈ Interval(0.0, 1.0)] - -input_ = length(domains) -n = 7 -chain = [ - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)), - Lux.Chain(Lux.Dense(input_, n, Lux.tanh), Lux.Dense(n, n, Lux.tanh), - Lux.Dense(n, 1)) -] - -#Generate Data -function lorenz!(du, u, p, t) - du[1] = 10.0 * (u[2] - u[1]) - du[2] = u[1] * (28.0 - u[3]) - u[2] - du[3] = u[1] * u[2] - (8 / 3) * u[3] -end - -u0 = [1.0; 0.0; 0.0] -tspan = (0.0, 1.0) -prob = ODEProblem(lorenz!, u0, tspan) -sol = solve(prob, Tsit5(), dt = 0.01, saveat = 0.05) -ts = sol.t -us = hcat(sol.u...) -us = us .+ ((0.05 .* randn(size(us))) .* us) -ts_ = hcat(sol(ts).t...)[1, :] -dataset = [hcat(us[i, :], ts_) for i in 1:3] - -discretization = BayesianPINN(chain, GridTraining([0.01]); param_estim = true, - dataset = [dataset, nothing]) - -@named pde_system = PDESystem(eqs, bcs, domains, - [t], [x(t), y(t), z(t)], [σ_], defaults = Dict([p => 1.0 for p in [σ_]])) - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 100, - bcstd = [0.3, 0.3, 0.3], - phystd = [0.1, 0.1, 0.1], - l2std = [1, 1, 1], - priorsNNw = (0.0, 1.0), - saveats = [0.01], - param = [Normal(14.0, 2)], progress = true) - -idealp = 10.0 -p_ = sol1.estimated_de_params[1] -@test sum(abs, pmean(p_) - 10.00) < 0.3 * idealp[1] -# @test sum(abs, pmean(p_[2]) - (8 / 3)) < 0.3 * idealp[2] - -@parameters x y -@variables u(..) -Dxx = Differential(x)^2 -Dyy = Differential(y)^2 - -# 2D PDE -eq = Dxx(u(x, y)) + Dyy(u(x, y)) ~ -sin(pi * x) * sin(pi * y) - -# Boundary conditions -bcs = [u(0, y) ~ 0.0, u(1, y) ~ 0.0, - u(x, 0) ~ 0.0, u(x, 1) ~ 0.0] - -# Space and time domains -domains = [x ∈ Interval(0.0, 1.0), - y ∈ Interval(0.0, 1.0)] - -# Neural network -dim = 2 # number of dimensions -chain = Lux.Chain(Lux.Dense(dim, 9, Lux.σ), Lux.Dense(9, 9, Lux.σ), Lux.Dense(9, 1)) - -# Discretization -dx = 0.04 -discretization = BayesianPINN([chain], GridTraining(dx), dataset = [[dataset], nothing]) - -@named pde_system = PDESystem(eq, bcs, domains, [x, y], [u(x, y)]) - -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 5, - bcstd = [0.01, 0.01, 0.01, 0.01], - phystd = [0.005], - priorsNNw = (0.0, 2.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -xs = sol1.timepoints[1] -sol1.ensemblesol[1] -analytic_sol_func(x, y) = (sin(pi * x) * sin(pi * y)) / (2pi^2) - -dataset = hcat(u_real, xs') -u_predict = pmean(sol1.ensemblesol[1]) -u_real = [analytic_sol_func(xs[:, i][1], xs[:, i][2]) for i in 1:length(xs[1, :])] -@test u_predict≈u_real atol=0.8 - -# KS EQUATION -using NeuralPDE, Flux, Lux, ModelingToolkit, LinearAlgebra, AdvancedHMC -import ModelingToolkit: Interval, infimum, supremum, Distributions -using Plots, MonteCarloMeasurements, StatsPlots -# plotly() - -@parameters x, t, α -@variables u(..) -Dt = Differential(t) -Dx = Differential(x) -Dx2 = Differential(x)^2 -Dx3 = Differential(x)^3 -Dx4 = Differential(x)^4 - -# α = 1 -β = 4 -γ = 1 -eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 - -u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 - -bcs = [u(x, 0) ~ u_analytic(x, 0), - u(-10, t) ~ u_analytic(-10, t), - u(10, t) ~ u_analytic(10, t), - Dx(u(-10, t)) ~ du(-10, t), - Dx(u(10, t)) ~ du(10, t)] - -# Space and time domains -domains = [x ∈ Interval(-10.0, 10.0), - t ∈ Interval(0.0, 1.0)] - -# Discretization -dx = 0.4; -dt = 0.2; - -# Function to compute analytical solution at a specific point (x, t) -function u_analytic_point(x, t) - z = -x / 2 + t - return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 -end - -# Function to generate the dataset matrix -function generate_dataset_matrix(domains, dx, dt, xlim, tlim) - x_values = xlim[1]:dx:xlim[2] - t_values = tlim[1]:dt:tlim[2] - - dataset = [] - - for t in t_values - for x in x_values - u_value = u_analytic_point(x, t) - push!(dataset, [u_value, x, t]) - end - end - - return vcat([data' for data in dataset]...) -end - -datasetpde = [generate_dataset_matrix(domains, dx, dt, [-10, 10], [0.0, 1.0])] -datasetpde_new = [generate_dataset_matrix(domains, dx, dt, [-10, 0], [0.0, 1.0])] - -# noise to dataset -noisydataset = deepcopy(datasetpde) -noisydataset[1][:, 1] = noisydataset[1][:, 1] .+ (randn(size(noisydataset[1][:, 1])) .* 0.8) - -noisydataset_new = deepcopy(datasetpde_new) -noisydataset_new[1][:, 1] = noisydataset_new[1][:, 1] .+ - (randn(size(noisydataset_new[1][:, 1])) .* 0.8) - -plot(datasetpde[1][:, 2], datasetpde[1][:, 1], title = "Dataset from Analytical Solution") -scatter!(noisydataset[1][:, 2], noisydataset[1][:, 1]) - -plot(datasetpde[1][:, 2], datasetpde[1][:, 3], datasetpde[1][:, 1], - title = "Dataset from Analytical Solution") -scatter!(noisydataset[1][:, 2], noisydataset[1][:, 3], noisydataset[1][:, 1]) - -plot(datasetpde_new[1][:, 2], datasetpde_new[1][:, 1], - title = "Dataset from Analytical Solution") -scatter!(noisydataset_new[1][:, 2], noisydataset_new[1][:, 1]) - -plot(datasetpde_new[1][:, 2], datasetpde_new[1][:, 3], - datasetpde_new[1][:, 1], title = "Dataset from Analytical Solution") -scatter!(noisydataset_new[1][:, 2], noisydataset_new[1][:, 3], noisydataset_new[1][:, 1]) - -noise_std = 1.4 -original_data = datasetpde[1][:, 1] -original_std = std(original_data) -ratio = noise_std / original_std - -# Neural network -chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), - Lux.Dense(8, 8, Lux.tanh), - Lux.Dense(8, 1)) - -discretization = NeuralPDE.BayesianPINN([chain], - GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset, nothing]) - -discretization_new = NeuralPDE.BayesianPINN([chain], - GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset_new, nothing]) - -@named pde_system = PDESystem(eq, - bcs, - domains, - [x, t], - [u(x, t)], - [α], - defaults = Dict([α => 2.0])) - -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.5], - phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol1_1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 90, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.7], - phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -sol3 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 100, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.7], l2std = [0.15], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 3.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -sol4 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 80, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.7], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1_1.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol1_1.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol2.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol2.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol3.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol3.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol4.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - first(pmean(phi([x, t], sol4.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -sol0_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], - phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -julia > sol5_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 170, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -sol1_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], - phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol1_1_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 110, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.5], - phystd = [0.5], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol1_2_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 150, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.1], - phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 3.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol1_3_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 150, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], - phystd = [0.3], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 3.0), - saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) - -sol2_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 140, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.2], l2std = [0.2], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -sol3_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 140, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -sol4_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 160, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -sol5_new = ahmc_bayesian_pinn_pde(pde_system, - discretization_new; - draw_samples = 170, Kernel = AdvancedHMC.NUTS(0.8), - bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], - phystd = [0.2], l2std = [0.1], param = [Distributions.Normal(2.0, 2)], - priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol0_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - - first(pmean(phi([x, t], sol0_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - - first(pmean(phi([x, t], sol1_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1_1_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - - first(pmean(phi([x, t], sol1_1_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1_2_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - - first(pmean(phi([x, t], sol1_2_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol1_3_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - - first(pmean(phi([x, t], sol1_3_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol2_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - - first(pmean(phi([x, t], sol2_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol3_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - - first(pmean(phi([x, t], sol3_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol4_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - - first(pmean(phi([x, t], sol4_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol5_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - - first(pmean(phi([x, t], sol5_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] - -p1 = plot(xs, u_predict, title = "predict") -p2 = plot(xs, u_real, title = "analytic") -p3 = plot(xs, diff_u, title = "error") -plot(p1, p2, p3) - -phi = discretization_new.phi[1] -xs, ts = [infimum(d.domain):dx:supremum(d.domain) - for (d, dx) in zip(domains, [dx / 10, dt])] - -u_predict = [[first(pmean(phi([x, t], sol5_new.estimated_nn_params[1]))) for x in xs] - for t in ts] -u_real = [[u_analytic(x, t) for x in xs] for t in ts] -diff_u = [[abs(u_analytic(x, t) - - first(pmean(phi([x, t], sol5_new.estimated_nn_params[1])))) - for x in xs] - for t in ts] -p1 = plot(ts, xs, u_predict, title = "predict") -p2 = plot(ts, xs, u_real, title = "analytic") -p3 = plot(ts, xs, diff_u, title = "error") -plot(p1, p2, p3) - -# MCMC chain analysis -plot(sol1.original.mcmc_chain) -plot(sol2.original.mcmc_chain) - -plot(sol0_new.original.mcmc_chain) -plot(sol2_new.original.mcmc_chain) - -plot(sol1.original.mcmc_chain) -meanplot(sol1.original.mcmc_chain) -autocorplot(sol1.original.mcmc_chain) -traceplot(sol1.original.mcmc_chain) - -plot(sol2.original.mcmc_chain) -meanplot(sol2.original.mcmc_chain) -autocorplot(sol2.original.mcmc_chain) -traceplot(sol2.original.mcmc_chain) - -plot(sol0_new.original.mcmc_chain) -meanplot(sol0_new.original.mcmc_chain) -autocorplot(sol0_new.original.mcmc_chain) - -plot(sol2_new.original.mcmc_chain) -meanplot(sol2_new.original.mcmc_chain) -autocorplot(sol2_new.original.mcmc_chain) - -plot(sol3_new.original.mcmc_chain) -meanplot(sol3_new.original.mcmc_chain) -autocorplot(sol3_new.original.mcmc_chain) \ No newline at end of file From ac251447db2eff4e47e31be92f7f95c44330a4b5 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Wed, 2 Oct 2024 19:12:45 +0530 Subject: [PATCH 46/53] corrections in rebase --- src/BPINN_ode.jl | 6 +----- src/PDE_BPINN.jl | 2 +- src/advancedHMC_MCMC.jl | 1 - test/BPINN_Tests.jl | 4 ++-- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/BPINN_ode.jl b/src/BPINN_ode.jl index 5b413f9520..9960006b18 100644 --- a/src/BPINN_ode.jl +++ b/src/BPINN_ode.jl @@ -102,8 +102,6 @@ struct BNNODE{C, K, IT <: NamedTuple, Integratorkwargs::IT numensemble::Int64 estim_collocate::Bool - numensemble::Int64 - estim_collocate::Bool autodiff::Bool progress::Bool verbose::Bool @@ -116,8 +114,6 @@ function BNNODE(chain, Kernel = HMC; strategy = nothing, draw_samples = 2000, Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), Integratorkwargs = (Integrator = Leapfrog,), - numensemble = floor(Int, draw_samples / 3), - estim_collocate = false, numensemble = floor(Int, draw_samples / 3), estim_collocate = false, autodiff = false, progress = false, verbose = false) @@ -195,7 +191,7 @@ function SciMLBase.__solve(prob::SciMLBase.ODEProblem, @unpack chain, l2std, phystd, param, priorsNNw, Kernel, strategy, draw_samples, dataset, init_params, nchains, physdt, Adaptorkwargs, Integratorkwargs, - MCMCkwargs, numensemble, estim_collocate, numensemble, estim_collocate, autodiff, progress, + MCMCkwargs, numensemble, estim_collocate, autodiff, progress, verbose = alg # ahmc_bayesian_pinn_ode needs param=[] for easier vcat operation for full vector of parameters diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index 615f801bc5..c1cd182d98 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -471,7 +471,7 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; # append Ode params to all paramvector - initial_θ if ninv > 0 # shift ode params(initialise ode params by prior means) - # check if means or user speified is better + # check if means or user specified is better initial_θ = vcat(initial_θ, [Distributions.params(param[i])[1] for i in 1:ninv]) priors = vcat(priors, param) nparameters += ninv diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index d72e13d229..7105346aa0 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -17,7 +17,6 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, extraparams::Int init_params::I estim_collocate::Bool - estim_collocate::Bool function LogTargetDensity(dim, prob, chain::Optimisers.Restructure, st, strategy, dataset, diff --git a/test/BPINN_Tests.jl b/test/BPINN_Tests.jl index 6534e88409..88e794df89 100644 --- a/test/BPINN_Tests.jl +++ b/test/BPINN_Tests.jl @@ -145,7 +145,7 @@ end dataset = [x̂, time] physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - # seperate set of points for testing the solve() call (it uses saveat 1/50 hence here length 501) + # separate set of points for testing the solve() call (it uses saveat 1/50 hence here length 501) time1 = vec(collect(Float64, range(tspan[1], tspan[2], length = 501))) physsol2 = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] @@ -264,7 +264,7 @@ end dataset = [x̂, time] physsol1 = [linear_analytic(prob.u0, p, time[i]) for i in eachindex(time)] - # seperate set of points for testing the solve() call (it uses saveat 1/50 hence here length 501) + # separate set of points for testing the solve() call (it uses saveat 1/50 hence here length 501) time1 = vec(collect(Float64, range(tspan[1], tspan[2], length = 501))) physsol2 = [linear_analytic(prob.u0, p, time1[i]) for i in eachindex(time1)] From 2ea5212e0c0c1f2bb49c10010b7f37ff6e77a68d Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Fri, 11 Oct 2024 16:14:10 +0530 Subject: [PATCH 47/53] tests pass locally --- test/BPINN_PDEinvsol_tests.jl | 248 +++++++++++++++++++++------------- 1 file changed, 151 insertions(+), 97 deletions(-) diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index b82502259b..07ce051b3a 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -34,33 +34,7 @@ Random.seed!(100) u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) dataset = [hcat(u1, timepoints)] - # TODO: correct implementations - # # checking all training strategies - # discretization = BayesianPINN([chainl], StochasticTraining(200), param_estim = true, - # dataset = [dataset, nothing]) - - # ahmc_bayesian_pinn_pde(pde_system, - # discretization; - # draw_samples = 1500, - # bcstd = [0.05], - # phystd = [0.01], l2std = [0.01], - # priorsNNw = (0.0, 1.0), - # saveats = [1 / 50.0], - # param = [LogNormal(6.0, 0.5)]) - - # discretization = BayesianPINN([chainl], QuasiRandomTraining(200), param_estim = true, - # dataset = [dataset, nothing]) - - # ahmc_bayesian_pinn_pde(pde_system, - # discretization; - # draw_samples = 1500, - # bcstd = [0.05], - # phystd = [0.01], l2std = [0.01], - # priorsNNw = (0.0, 1.0), - # saveats = [1 / 50.0], - # param = [LogNormal(6.0, 0.5)]) - - # alternative to QuadratureTraining [WIP] + # TODO: correct BPINN implementations for Training Strategies. discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, dataset = [dataset, nothing]) @@ -164,73 +138,153 @@ function recur_expression(exp, Dict_differentials) end end -println("Example 3: 2D Periodic System with New parameter estimation") -@parameters t, p -@variables u(..) - -Dt = Differential(t) -eqs = Dt(u(t)) - cos(p * t) * u(t) ~ 0 -bcs = [u(0) ~ 0.0] -domains = [t ∈ Interval(0.0, 2.0)] - -chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) -initl, st = Lux.setup(Random.default_rng(), chainl) - -@named pde_system = PDESystem(eqs, - bcs, - domains, - [t], - [u(t)], - [p], - defaults = Dict([p => 4.0])) - -analytic_sol_func1(u0, t) = u0 + sin(2 * π * t) / (2 * π) -timepoints = collect(0.0:(1 / 100.0):2.0) -u1 = [analytic_sol_func1(0.0, timepoint) for timepoint in timepoints] -u1 = u1 .+ (u1 .* 0.2) .* randn(size(u1)) -dataset = [hcat(u1, timepoints)] - -discretization = BayesianPINN([chainl], GridTraining([0.02]), param_estim = true, - dataset = [dataset, nothing]) - -# creating dictionary for masking equations -eqs = pde_system.eqs -Dict_differentials = Dict() -exps = toexpr.(eqs) -nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] - -sol1 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], phystdnew = [0.05], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)], - Dict_differentials = Dict_differentials) - -sol2 = ahmc_bayesian_pinn_pde(pde_system, - discretization; - draw_samples = 1500, - bcstd = [0.05], - phystd = [0.01], l2std = [0.01], - priorsNNw = (0.0, 1.0), - saveats = [1 / 50.0], - param = [LogNormal(6.0, 0.5)]) - -param = 2 * π -ts = vec(sol1.timepoints[1]) -u_real = [analytic_sol_func1(0.0, t) for t in ts] -u_predict = pmean(sol1.ensemblesol[1]) - -@test u_predict≈u_real atol=1.5 -@test mean(u_predict .- u_real) < 0.1 -@test sol1.estimated_de_params[1]≈param atol=param * 0.3 - -ts = vec(sol2.timepoints[1]) -u_real = [analytic_sol_func1(0.0, t) for t in ts] -u_predict = pmean(sol2.ensemblesol[1]) - -@test u_predict≈u_real atol=1.5 -@test mean(u_predict .- u_real) < 0.1 -@test sol1.estimated_de_params[1]≈param atol=param * 0.3 \ No newline at end of file +@testset "improvement in Solving Parametric Kuromo-Sivashinsky Equation" begin + @parameters x, t, α + @variables u(..) + Dt = Differential(t) + Dx = Differential(x) + Dx2 = Differential(x)^2 + Dx3 = Differential(x)^3 + Dx4 = Differential(x)^4 + + # α = 1 (KS equation to be parametric in a) + β = 4 + γ = 1 + eq = Dt(u(x, t)) + u(x, t) * Dx(u(x, t)) + α * Dx2(u(x, t)) + β * Dx3(u(x, t)) + γ * Dx4(u(x, t)) ~ 0 + + u_analytic(x, t; z = -x / 2 + t) = 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 + du(x, t; z = -x / 2 + t) = 15 / 2 * (tanh(z) + 1) * (3 * tanh(z) - 1) * sech(z)^2 + + bcs = [u(x, 0) ~ u_analytic(x, 0), + u(-10, t) ~ u_analytic(-10, t), + u(10, t) ~ u_analytic(10, t), + Dx(u(-10, t)) ~ du(-10, t), + Dx(u(10, t)) ~ du(10, t)] + + # Space and time domains + domains = [x ∈ Interval(-10.0, 10.0), + t ∈ Interval(0.0, 1.0)] + + # Discretization + dx = 0.4 + dt = 0.2 + + # Function to compute analytical solution at a specific point (x, t) + function u_analytic_point(x, t) + z = -x / 2 + t + return 11 + 15 * tanh(z) - 15 * tanh(z)^2 - 15 * tanh(z)^3 + end + + # Function to generate the dataset matrix + function generate_dataset_matrix(domains, dx, dt, xlim, tlim) + x_values = xlim[1]:dx:xlim[2] + t_values = tlim[1]:dt:tlim[2] + + dataset = [] + + for t in t_values + for x in x_values + u_value = u_analytic_point(x, t) + push!(dataset, [u_value, x, t]) + end + end + + return vcat([data' for data in dataset]...) + end + + # considering sparse dataset from half of x's domain + datasetpde_new = [generate_dataset_matrix(domains, dx, dt, [-10, 0], [0.0, 1.0])] + + # Adding Gaussian noise with a 0.8 std + noisydataset_new = deepcopy(datasetpde_new) + noisydataset_new[1][:, 1] = noisydataset_new[1][:, 1] .+ + (randn(size(noisydataset_new[1][:, 1])) .* 0.8) + + # Neural network + chain = Lux.Chain(Lux.Dense(2, 8, Lux.tanh), + Lux.Dense(8, 8, Lux.tanh), + Lux.Dense(8, 1)) + + # Discretization for old and new models + discretization = NeuralPDE.BayesianPINN([chain], + GridTraining([dx, dt]), param_estim = true, dataset = [noisydataset_new, nothing]) + + # let α default to 2.0 + @named pde_system = PDESystem(eq, + bcs, + domains, + [x, t], + [u(x, t)], + [α], + defaults = Dict([α => 2.0])) + + # neccesarry for loss function contruction (involves Operator masking) + eqs = pde_system.eqs + Dict_differentials = Dict() + exps = toexpr.(eqs) + nullobj = [recur_expression(exp, Dict_differentials) for exp in exps] + + # Dict_differentials is now ; + # Dict{Any, Any} with 5 entries: + # Differential(x)(Differential(x)(u(x, t))) => diff_5 + # Differential(x)(Differential(x)(Differential(x)(u(x… => diff_1 + # Differential(x)(Differential(x)(Differential(x)(Dif… => diff_2 + # Differential(x)(u(x, t)) => diff_4 + # Differential(t)(u(x, t)) => diff_3 + + # using HMC algorithm due to convergence, stability, time of training. (refer to mcmc chain plots) + # choice of std for objectives is very important + # pass in Dict_differentials, phystdnew arguments when using the new model + + sol_new = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 150, + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystdnew = [0.2], + phystd = [0.2], l2std = [0.5], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + Dict_differentials = Dict_differentials, + progress = true) + + sol_old = ahmc_bayesian_pinn_pde(pde_system, + discretization; + draw_samples = 150, + bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], + phystd = [0.2], l2std = [0.5], param = [Distributions.Normal(2.0, 2)], + priorsNNw = (0.0, 1.0), + saveats = [1 / 100.0, 1 / 100.0], + progress = true) + + phi = discretization.phi[1] + xs, ts = [infimum(d.domain):dx:supremum(d.domain) + for (d, dx) in zip(domains, [dx / 10, dt])] + u_real = [[u_analytic(x, t) for x in xs] for t in ts] + + u_predict_new = [[first(pmean(phi([x, t], sol_new.estimated_nn_params[1]))) for x in xs] + for t in ts] + + diff_u_new = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol_new.estimated_nn_params[1])))) + for x in xs] + for t in ts] + + u_predict_old = [[first(pmean(phi([x, t], sol_old.estimated_nn_params[1]))) for x in xs] + for t in ts] + diff_u_old = [[abs(u_analytic(x, t) - + first(pmean(phi([x, t], sol_old.estimated_nn_params[1])))) + for x in xs] + for t in ts] + + @test all(all, [((diff_u_new[i]) .^ 2 .< 0.5) for i in 1:6]) == true + @test all(all, [((diff_u_old[i]) .^ 2 .< 0.5) for i in 1:6]) == false + + MSE_new = [sum(abs2, diff_u_new[i]) for i in 1:6] + MSE_old = [sum(abs2, diff_u_old[i]) for i in 1:6] + @test (MSE_new .< MSE_old) == [1, 1, 1, 1, 1, 1] + + param_new = sol_new.estimated_de_params[1] + param_old = sol_old.estimated_de_params[1] + α = 1 + @test abs(param_new - α) < 0.2 * α + @test abs(param_new - α) < abs(param_old - α) +end \ No newline at end of file From 9fff415c6dc00a8c1a4bcfa5bfd00a982e8c98c8 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Fri, 11 Oct 2024 16:20:19 +0530 Subject: [PATCH 48/53] spell checks, Statistics.jl vers --- Project.toml | 2 +- test/BPINN_PDEinvsol_tests.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 026a29ba72..32d95d792b 100644 --- a/Project.toml +++ b/Project.toml @@ -75,7 +75,7 @@ Reexport = "1.2" RuntimeGeneratedFunctions = "0.5.12" SafeTestsets = "0.1" SciMLBase = "2.28" -Statistics = "1.10" +Statistics = "1.11" SymbolicUtils = "1.5, 2, 3" Symbolics = "5.27.1, 6" Test = "1" diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 07ce051b3a..1d8dca0e1f 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -218,7 +218,7 @@ end [α], defaults = Dict([α => 2.0])) - # neccesarry for loss function contruction (involves Operator masking) + # neccesarry for loss function construction (involves Operator masking) eqs = pde_system.eqs Dict_differentials = Dict() exps = toexpr.(eqs) From 247b8e3f7a7f2ca11314dab12cd3e303b22fcdff Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Sat, 12 Oct 2024 07:09:28 +0530 Subject: [PATCH 49/53] update tests --- src/PDE_BPINN.jl | 2 +- test/BPINN_PDE_tests.jl | 24 +++++++++++++----------- test/BPINN_PDEinvsol_tests.jl | 6 ++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index c1cd182d98..b03f158942 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -416,7 +416,7 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; end end - # [WIP] add overall functionality for BC dataset points + # [WIP] add overall functionality for BC dataset points (case of parametric BC) if ((dataset_bc isa Nothing) && (dataset_pde isa Nothing)) dataset = nothing elseif dataset_bc isa Nothing diff --git a/test/BPINN_PDE_tests.jl b/test/BPINN_PDE_tests.jl index 0cefaba18f..35d62bd352 100644 --- a/test/BPINN_PDE_tests.jl +++ b/test/BPINN_PDE_tests.jl @@ -16,7 +16,7 @@ Random.seed!(100) eqs = Dt(u(t)) - cos(2 * π * t) ~ 0 bcs = [u(0) ~ 0.0] domains = [t ∈ Interval(0.0, 2.0)] - chainl = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 1)) + chainl = Lux.Chain(Lux.Dense(1, 5, tanh), Lux.Dense(5, 5, tanh), Lux.Dense(5, 1)) initl, st = Lux.setup(Random.default_rng(), chainl) @named pde_system = PDESystem(eqs, bcs, domains, [t], [u(t)]) @@ -25,10 +25,10 @@ Random.seed!(100) sol1 = ahmc_bayesian_pinn_pde(pde_system, discretization; - draw_samples = 2000, - bcstd = [0.02], + draw_samples = 250, + bcstd = [0.001], phystd = [0.01], - priorsNNw = (0.0, 10.0), + priorsNNw = (0.0, 1.0), saveats = [1 / 50.0]) analytic_sol_func(u0, t) = u0 + sin(2 * π * t) / (2 * π) @@ -36,8 +36,8 @@ Random.seed!(100) u_real = [analytic_sol_func(0.0, t) for t in ts] u_predict = pmean(sol1.ensemblesol[1]) - @test u_predict≈u_real atol=0.05 - @test mean(u_predict .- u_real) < 1e-3 + @test u_predict≈u_real atol=0.02 + @test mean(abs.(u_predict .- u_real)) < 1e-3 end @testset "Example 2: 1D ODE" begin @@ -159,10 +159,10 @@ end sol1 = ahmc_bayesian_pinn_pde(pde_system, discretization; - draw_samples = 200, - bcstd = [0.01, 0.01, 0.01, 0.01], - phystd = [0.005], - priorsNNw = (0.0, 2.0), + draw_samples = 400, + bcstd = [0.05, 0.05, 0.05, 0.05], + phystd = [0.05], + priorsNNw = (0.0, 1.0), saveats = [1 / 100.0, 1 / 100.0]) xs = sol1.timepoints[1] @@ -170,7 +170,9 @@ end u_predict = pmean(sol1.ensemblesol[1]) u_real = [analytic_sol_func(xs[:, i][1], xs[:, i][2]) for i in 1:length(xs[1, :])] - @test u_predict≈u_real atol=0.8 + + @test sum(abs2.(u_predict .- u_real)) < 0.1 + @test u_predict≈u_real atol=0.1 end @testset "Translating from Flux" begin diff --git a/test/BPINN_PDEinvsol_tests.jl b/test/BPINN_PDEinvsol_tests.jl index 1d8dca0e1f..5cc53df354 100644 --- a/test/BPINN_PDEinvsol_tests.jl +++ b/test/BPINN_PDEinvsol_tests.jl @@ -243,8 +243,7 @@ end phystd = [0.2], l2std = [0.5], param = [Distributions.Normal(2.0, 2)], priorsNNw = (0.0, 1.0), saveats = [1 / 100.0, 1 / 100.0], - Dict_differentials = Dict_differentials, - progress = true) + Dict_differentials = Dict_differentials) sol_old = ahmc_bayesian_pinn_pde(pde_system, discretization; @@ -252,8 +251,7 @@ end bcstd = [0.1, 0.1, 0.1, 0.1, 0.1], phystd = [0.2], l2std = [0.5], param = [Distributions.Normal(2.0, 2)], priorsNNw = (0.0, 1.0), - saveats = [1 / 100.0, 1 / 100.0], - progress = true) + saveats = [1 / 100.0, 1 / 100.0]) phi = discretization.phi[1] xs, ts = [infimum(d.domain):dx:supremum(d.domain) From f81aa7a10dfd9e4ed0be0721a4d1f399933904d5 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Tue, 15 Oct 2024 17:49:28 +0530 Subject: [PATCH 50/53] low level changes, transform fixes --- src/BPINN_ode.jl | 13 +++--- src/PDE_BPINN.jl | 8 ++-- src/advancedHMC_MCMC.jl | 34 +++++++++------ src/pinn_types.jl | 2 +- test/BPINN_PDE_tests.jl | 11 ++--- test/BPINN_Tests.jl | 97 ++++++++--------------------------------- 6 files changed, 55 insertions(+), 110 deletions(-) diff --git a/src/BPINN_ode.jl b/src/BPINN_ode.jl index 9960006b18..39bb0aac72 100644 --- a/src/BPINN_ode.jl +++ b/src/BPINN_ode.jl @@ -93,6 +93,7 @@ struct BNNODE{C, K, IT <: NamedTuple, param::P l2std::Vector{Float64} phystd::Vector{Float64} + phynewstd::Vector{Float64} dataset::D physdt::Float64 MCMCkwargs::H @@ -107,7 +108,7 @@ struct BNNODE{C, K, IT <: NamedTuple, verbose::Bool end function BNNODE(chain, Kernel = HMC; strategy = nothing, draw_samples = 2000, - priorsNNw = (0.0, 2.0), param = nothing, l2std = [0.05], phystd = [0.05], + priorsNNw = (0.0, 2.0), param = nothing, l2std = [0.05], phystd = [0.05], phynewstd = [0.05], dataset = [nothing], physdt = 1 / 20.0, MCMCkwargs = (n_leapfrog = 30,), nchains = 1, init_params = nothing, Adaptorkwargs = (Adaptor = StanHMCAdaptor, @@ -121,7 +122,7 @@ function BNNODE(chain, Kernel = HMC; strategy = nothing, draw_samples = 2000, (chain = adapt(FromFluxAdaptor(false, false), chain)) BNNODE(chain, Kernel, strategy, draw_samples, priorsNNw, param, l2std, - phystd, dataset, physdt, MCMCkwargs, + phystd, phynewstd, dataset, physdt, MCMCkwargs, nchains, init_params, Adaptorkwargs, Integratorkwargs, numensemble, estim_collocate, @@ -186,9 +187,8 @@ function SciMLBase.__solve(prob::SciMLBase.ODEProblem, reltol = 1.0f-3, verbose = false, saveat = 1 / 50.0, - maxiters = nothing, - numensemble = floor(Int, alg.draw_samples / 3)) - @unpack chain, l2std, phystd, param, priorsNNw, Kernel, strategy, + maxiters = nothing,) + @unpack chain, l2std, phystd, phynewstd, param, priorsNNw, Kernel, strategy, draw_samples, dataset, init_params, nchains, physdt, Adaptorkwargs, Integratorkwargs, MCMCkwargs, numensemble, estim_collocate, autodiff, progress, @@ -206,7 +206,8 @@ function SciMLBase.__solve(prob::SciMLBase.ODEProblem, strategy = strategy, dataset = dataset, draw_samples = draw_samples, init_params = init_params, - physdt = physdt, l2std = l2std, + physdt = physdt, phynewstd = phynewstd, + l2std = l2std, phystd = phystd, priorsNNw = priorsNNw, param = param, diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index b03f158942..044080118e 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -562,14 +562,14 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; mcmc_chain = MCMCChains.Chains(matrix_samples') @info("Sampling Complete.") - @info("Current Physics Log-likelihood : ", + @info("Final Physics Log-likelihood : ", ℓπ.full_loglikelihood(setparameters(ℓπ, samples[end]), ℓπ.allstd)) - @info("Current Prior Log-likelihood : ", priorlogpdf(ℓπ, samples[end])) - @info("Current MSE against dataset Log-likelihood : ", + @info("Final Prior Log-likelihood : ", priorlogpdf(ℓπ, samples[end])) + @info("Final MSE against dataset Log-likelihood : ", L2LossData(ℓπ, samples[end])) if !(newloss isa Nothing) - @info("Current L2_LOSSY : ", + @info("Final L2_LOSSY : ", ℓπ.L2_loss2(setparameters(ℓπ, samples[end]), ℓπ.allstd)) end diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index 7105346aa0..8b996fce5c 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -11,6 +11,7 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, dataset::D priors::P phystd::Vector{Float64} + phynewstd::Vector{Float64} l2std::Vector{Float64} autodiff::Bool physdt::Float64 @@ -20,7 +21,7 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, function LogTargetDensity(dim, prob, chain::Optimisers.Restructure, st, strategy, dataset, - priors, phystd, l2std, autodiff, physdt, extraparams, + priors, phystd, phynewstd, l2std, autodiff, physdt, extraparams, init_params::AbstractVector, estim_collocate) new{ typeof(chain), @@ -36,6 +37,7 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, dataset, priors, phystd, + phynewstd, l2std, autodiff, physdt, @@ -45,7 +47,7 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, end function LogTargetDensity(dim, prob, chain::Lux.AbstractExplicitLayer, st, strategy, dataset, - priors, phystd, l2std, autodiff, physdt, extraparams, + priors, phystd, phynewstd, l2std, autodiff, physdt, extraparams, init_params::NamedTuple, estim_collocate) new{ typeof(chain), @@ -58,7 +60,8 @@ mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, prob, chain, st, strategy, dataset, priors, - phystd, l2std, + phystd, phynewstd, + l2std, autodiff, physdt, extraparams, @@ -136,10 +139,10 @@ function L2loss2(Tar::LogTargetDensity, θ) physlogprob = 0 for i in 1:length(Tar.prob.u0) - # can add phystd[i] for u[i] + # can add phystdnew[i] for u[i] physlogprob += logpdf(MvNormal(deri_physsol[i, :], LinearAlgebra.Diagonal(map(abs2, - (Tar.l2std[i] * 4.0) .* + (Tar.phynewstd[i]) .* ones(length(nnsol[i, :]))))), nnsol[i, :]) end @@ -162,7 +165,7 @@ function L2LossData(Tar::LogTargetDensity, θ) L2logprob = 0 for i in 1:length(Tar.prob.u0) - # for u[i] ith vector must be added to dataset,nn[1,:] is the dx in lotka_volterra + # for u[i] ith vector must be added to dataset, nn[1,:] is the dx in lotka_volterra L2logprob += logpdf( MvNormal(nn[i, :], LinearAlgebra.Diagonal(abs2.(Tar.l2std[i] .* @@ -395,7 +398,7 @@ end ahmc_bayesian_pinn_ode(prob, chain; strategy = GridTraining, dataset = [nothing],init_params = nothing, draw_samples = 1000, physdt = 1 / 20.0f0,l2std = [0.05], - phystd = [0.05], priorsNNw = (0.0, 2.0), + phystd = [0.05], phynewstd = [0.05], priorsNNw = (0.0, 2.0), param = [], nchains = 1, autodiff = false, Kernel = HMC, Adaptorkwargs = (Adaptor = StanHMCAdaptor, Metric = DiagEuclideanMetric, @@ -466,6 +469,7 @@ Incase you are only solving the Equations for solution, do not provide dataset * `draw_samples`: number of samples to be drawn in the MCMC algorithms (warmup samples are ~2/3 of draw samples) * `l2std`: standard deviation of BPINN prediction against L2 losses/Dataset * `phystd`: standard deviation of BPINN prediction against Chosen Underlying ODE System +* `phynewstd`: standard deviation of new loss func term * `priorsNNw`: Tuple of (mean, std) for BPINN Network parameters. Weights and Biases of BPINN are Normal Distributions by default. * `param`: Vector of chosen ODE parameters Distributions in case of Inverse problems. * `autodiff`: Boolean Value for choice of Derivative Backend(default is numerical) @@ -492,7 +496,7 @@ function ahmc_bayesian_pinn_ode(prob::SciMLBase.ODEProblem, chain; strategy = GridTraining, dataset = [nothing], init_params = nothing, draw_samples = 1000, physdt = 1 / 20.0, l2std = [0.05], - phystd = [0.05], priorsNNw = (0.0, 2.0), + phystd = [0.05], phynewstd = [0.05], priorsNNw = (0.0, 2.0), param = [], nchains = 1, autodiff = false, Kernel = HMC, Adaptorkwargs = (Adaptor = StanHMCAdaptor, @@ -558,7 +562,7 @@ function ahmc_bayesian_pinn_ode(prob::SciMLBase.ODEProblem, chain; t0 = prob.tspan[1] # dimensions would be total no of params,initial_nnθ for Lux namedTuples ℓπ = LogTargetDensity(nparameters, prob, recon, st, strategy, dataset, priors, - phystd, l2std, autodiff, physdt, ninv, initial_nnθ, estim_collocate) + phystd, phynewstd, l2std, autodiff, physdt, ninv, initial_nnθ, estim_collocate) try ℓπ(t0, initial_θ[1:(nparameters - ninv)]) @@ -574,7 +578,8 @@ function ahmc_bayesian_pinn_ode(prob::SciMLBase.ODEProblem, chain; @info("Current Prior Log-likelihood : ", priorweights(ℓπ, initial_θ)) @info("Current MSE against dataset Log-likelihood : ", L2LossData(ℓπ, initial_θ)) if estim_collocate - @info("Current gradient loss against dataset Log-likelihood : ", L2loss2(ℓπ, initial_θ)) + @info("Current gradient loss against dataset Log-likelihood : ", + L2loss2(ℓπ, initial_θ)) end Adaptor, Metric, targetacceptancerate = Adaptorkwargs[:Adaptor], @@ -624,11 +629,12 @@ function ahmc_bayesian_pinn_ode(prob::SciMLBase.ODEProblem, chain; adaptor; progress = progress, verbose = verbose) @info("Sampling Complete.") - @info("Current Physics Log-likelihood : ", physloglikelihood(ℓπ, samples[end])) - @info("Current Prior Log-likelihood : ", priorweights(ℓπ, samples[end])) - @info("Current MSE against dataset Log-likelihood : ", L2LossData(ℓπ, samples[end])) + @info("Final Physics Log-likelihood : ", physloglikelihood(ℓπ, samples[end])) + @info("Final Prior Log-likelihood : ", priorweights(ℓπ, samples[end])) + @info("Final MSE against dataset Log-likelihood : ", L2LossData(ℓπ, samples[end])) if estim_collocate - @info("Current gradient loss against dataset Log-likelihood : ", L2loss2(ℓπ, samples[end])) + @info("Final gradient loss against dataset Log-likelihood : ", + L2loss2(ℓπ, samples[end])) end # return a chain(basic chain),samples and stats diff --git a/src/pinn_types.jl b/src/pinn_types.jl index 59480d8a60..6944041efd 100644 --- a/src/pinn_types.jl +++ b/src/pinn_types.jl @@ -247,7 +247,7 @@ struct BayesianPINN{T, P, PH, DER, PE, AL, ADA, LOG, D, K} <: AbstractPINN multioutput = chain isa AbstractArray if multioutput !all(i -> i isa Lux.AbstractExplicitLayer, chain) && - (chain = Lux.transform.(chain)) + (chain = [adapt(FromFluxAdaptor(false, false), chain_i) for chain_i in chain]) else !(chain isa Lux.AbstractExplicitLayer) && (chain = adapt(FromFluxAdaptor(false, false), chain)) diff --git a/test/BPINN_PDE_tests.jl b/test/BPINN_PDE_tests.jl index 35d62bd352..2936911fe0 100644 --- a/test/BPINN_PDE_tests.jl +++ b/test/BPINN_PDE_tests.jl @@ -159,10 +159,10 @@ end sol1 = ahmc_bayesian_pinn_pde(pde_system, discretization; - draw_samples = 400, - bcstd = [0.05, 0.05, 0.05, 0.05], - phystd = [0.05], - priorsNNw = (0.0, 1.0), + draw_samples = 200, + bcstd = [0.0025, 0.0025, 0.0025, 0.0025], + phystd = [0.005], + priorsNNw = (0.0, 0.5), saveats = [1 / 100.0, 1 / 100.0]) xs = sol1.timepoints[1] @@ -171,8 +171,9 @@ end u_predict = pmean(sol1.ensemblesol[1]) u_real = [analytic_sol_func(xs[:, i][1], xs[:, i][2]) for i in 1:length(xs[1, :])] + @test mean(abs2.(u_predict .- u_real)) < 5e-3 + @test all(abs.(u_predict .- u_real) .< 15e-3) @test sum(abs2.(u_predict .- u_real)) < 0.1 - @test u_predict≈u_real atol=0.1 end @testset "Translating from Flux" begin diff --git a/test/BPINN_Tests.jl b/test/BPINN_Tests.jl index 88e794df89..1f5672d3f4 100644 --- a/test/BPINN_Tests.jl +++ b/test/BPINN_Tests.jl @@ -276,7 +276,7 @@ end dataset = dataset, draw_samples = 1000, l2std = [0.1], - phystd = [0.03], + phystd = [0.01], priorsNNw = (0.0, 1.0), param = [ @@ -288,7 +288,8 @@ end dataset = dataset, draw_samples = 1000, l2std = [0.1], - phystd = [0.03], + phystd = [0.01], + phynewstd = [0.01], priorsNNw = (0.0, 1.0), param = [ @@ -299,114 +300,50 @@ end dataset = dataset, draw_samples = 1000, l2std = [0.1], - phystd = [0.03], + phystd = [0.01], + phynewstd = [0.05], priorsNNw = (0.0, 1.0), param = [ Normal(-7, 3) - ], estim_collocate = true) + ], numensemble = 200, + estim_collocate = true) sol3lux_pestim = solve(prob, alg) # testing timepoints t = sol.t #------------------------------ ahmc_bayesian_pinn_ode() call - # Mean of last 500 sampled parameter's curves(lux chains)[Ensemble predictions] + # Mean of last 200 sampled parameter's curves(lux chains)[Ensemble predictions] θ = [vector_to_parameters(fhsampleslux12[i][1:(end - 1)], θinit) - for i in 750:length(fhsampleslux12)] + for i in 800:length(fhsampleslux12)] luxar = [chainlux12(t', θ[i], st)[1] for i in eachindex(θ)] luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] meanscurve2_1 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean θ = [vector_to_parameters(fhsampleslux22[i][1:(end - 1)], θinit) - for i in 750:length(fhsampleslux22)] + for i in 800:length(fhsampleslux22)] luxar = [chainlux12(t', θ[i], st)[1] for i in eachindex(θ)] luxmean = [mean(vcat(luxar...)[:, i]) for i in eachindex(t)] meanscurve2_2 = prob.u0 .+ (t .- prob.tspan[1]) .* luxmean - @test mean(abs.(sol.u .- meanscurve2_2)) < 6e-2 - @test mean(abs.(physsol1 .- meanscurve2_2)) < 6e-2 + @test mean(abs.(sol.u .- meanscurve2_2)) < 5e-2 + @test mean(abs.(physsol1 .- meanscurve2_2)) < 5e-2 @test mean(abs.(sol.u .- meanscurve2_1)) > mean(abs.(sol.u .- meanscurve2_2)) @test mean(abs.(physsol1 .- meanscurve2_1)) > mean(abs.(physsol1 .- meanscurve2_2)) # estimated parameters(lux chain) - param2 = mean(i[62] for i in fhsampleslux22[750:length(fhsampleslux22)]) - @test abs(param2 - p) < abs(0.25 * p) + param2 = mean(i[62] for i in fhsampleslux22[800:length(fhsampleslux22)]) + @test abs(param2 - p) < abs(0.2 * p) - param1 = mean(i[62] for i in fhsampleslux12[750:length(fhsampleslux12)]) - @test abs(param1 - p) < abs(0.75 * p) + param1 = mean(i[62] for i in fhsampleslux12[800:length(fhsampleslux12)]) + @test !(abs(param1 - p) < abs(0.2 * p)) @test abs(param2 - p) < abs(param1 - p) #-------------------------- solve() call # (lux chain) - @test mean(abs.(physsol2 .- pmean(sol3lux_pestim.ensemblesol[1]))) < 0.1 + @test mean(abs.(physsol2 .- pmean(sol3lux_pestim.ensemblesol[1]))) < 5e-2 # estimated parameters(lux chain) param3 = sol3lux_pestim.estimated_de_params[1] @test abs(param3 - p) < abs(0.2 * p) -end - -@testset "Example 4 - improvement" begin - function lotka_volterra(u, p, t) - # Model parameters. - α, β, γ, δ = p - # Current state. - x, y = u - - # Evaluate differential equations. - dx = (α - β * y) * x # prey - dy = (δ * x - γ) * y # predator - - return [dx, dy] - end - - # initial-value problem. - u0 = [1.0, 1.0] - p = [1.5, 1.0, 3.0, 1.0] - tspan = (0.0, 4.0) - prob = ODEProblem(lotka_volterra, u0, tspan, p) - - # Solve using OrdinaryDiffEq.jl solver - dt = 0.2 - solution = solve(prob, Tsit5(); saveat = dt) - - times = solution.t - u = hcat(solution.u...) - x = u[1, :] + (0.8 .* randn(length(u[1, :]))) - y = u[2, :] + (0.8 .* randn(length(u[2, :]))) - dataset = [x, y, times] - - chain = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), - Lux.Dense(6, 2)) - - alg1 = BNNODE(chain; - dataset = dataset, - draw_samples = 1000, - l2std = [0.2, 0.2], - phystd = [0.1, 0.1], - priorsNNw = (0.0, 1.0), - param = [ - Normal(2, 0.5), - Normal(2, 0.5), - Normal(2, 0.5), - Normal(2, 0.5)]) - - alg2 = BNNODE(chain; - dataset = dataset, - draw_samples = 1000, - l2std = [0.2, 0.2], - phystd = [0.1, 0.1], - priorsNNw = (0.0, 1.0), - param = [ - Normal(2, 0.5), - Normal(2, 0.5), - Normal(2, 0.5), - Normal(2, 0.5)], estim_collocate = true) - - @time sol_pestim1 = solve(prob, alg1; saveat = dt) - @time sol_pestim2 = solve(prob, alg2; saveat = dt) - - unsafe_comparisons(true) - bitvec = abs.(p .- sol_pestim1.estimated_de_params) .> - abs.(p .- sol_pestim2.estimated_de_params) - @test bitvec == ones(size(bitvec)) end \ No newline at end of file From 699fd7d6f9dd47614864899e360badcf0453f147 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Fri, 18 Oct 2024 13:53:23 +0530 Subject: [PATCH 51/53] changes from reviews --- src/PDE_BPINN.jl | 2 +- test/BPINN_Tests.jl | 167 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 166 insertions(+), 3 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index 044080118e..8957df889b 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -134,7 +134,7 @@ end function LogDensityProblems.logdensity(Tar::PDELogTargetDensity, θ) # for parameter estimation neccesarry to use multioutput case - if Tar.L2_loss2 isa Nothing + if Tar.L2_loss2 === nothing return Tar.full_loglikelihood(setparameters(Tar, θ), Tar.allstd) + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) else diff --git a/test/BPINN_Tests.jl b/test/BPINN_Tests.jl index 1f5672d3f4..657776bb29 100644 --- a/test/BPINN_Tests.jl +++ b/test/BPINN_Tests.jl @@ -6,7 +6,7 @@ using Statistics, Random, Functors, ComponentArrays using NeuralPDE, MonteCarloMeasurements using Flux -# note that current testing bounds can be easily further tightened but have been inflated for support for Julia build v1 +# note that current testing bounds can be further tightened but have been inflated for support for Julia build v1 # on latest Julia version it performs much better for below tests Random.seed!(100) @@ -346,4 +346,167 @@ end # estimated parameters(lux chain) param3 = sol3lux_pestim.estimated_de_params[1] @test abs(param3 - p) < abs(0.2 * p) -end \ No newline at end of file +end + +@testset "Example 4 - improvement" begin + function lotka_volterra(u, p, t) + # Model parameters. + α, β, γ, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (1 - β * y) * x * α # prey + dy = (δ * x - 1) * y * γ # predator + + return [dx, dy] + end + + # initial-value problem. + u0 = [1.0, 1.0] + p = [1.5, 2 / 3, 3.0, 1 / 3] + tspan = (0.0, 4.0) + prob = ODEProblem(lotka_volterra, u0, tspan, p) + + # Solve using OrdinaryDiffEq.jl solver + dt = 0.2 + solution = solve(prob, Tsit5(); saveat = dt) + + times = solution.t + u = hcat(solution.u...) + x = u[1, :] + (0.8 .* randn(length(u[1, :]))) + y = u[2, :] + (0.8 .* randn(length(u[2, :]))) + dataset = [x, y, times] + + chain = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), + Lux.Dense(6, 2)) + + alg1 = BNNODE(chain; + dataset = dataset, + draw_samples = 1000, + l2std = [0.05, 0.05], + phystd = [0.2, 0.2], + priorsNNw = (0.0, 1.0), + param = [ + Normal(2, 0.5), + Normal(2, 0.5), + Normal(2, 0.5), + Normal(2, 0.5)]) + + alg2 = BNNODE(chain; + dataset = dataset, + draw_samples = 1000, + l2std = [0.05, 0.05], + phystd = [0.2, 0.2], + phynewstd = [0.3, 0.1], + priorsNNw = (0.0, 1.0), + param = [ + Normal(2, 0.5), + Normal(2, 0.5), + Normal(2, 0.5), + Normal(2, 0.5)], estim_collocate = true) + + @time sol_pestim1 = solve(prob, alg1; saveat = dt) + @time sol_pestim2 = solve(prob, alg2; saveat = dt) + + unsafe_comparisons(true) + bitvec = abs.(p .- sol_pestim1.estimated_de_params) .> + abs.(p .- sol_pestim2.estimated_de_params) + @test bitvec == ones(size(bitvec)) +end + +function lotka_volterra(u, p, t) + # Model parameters. + β, δ = p + # Current state. + x, y = u + + # Evaluate differential equations. + dx = (3 - β * y) * x # prey + dy = (δ * x - 3) * y # predator + + return [dx, dy] +end + +# initial-value problem. +u0 = [1.0, 1.0] +p = [2, 1] +tspan = (0.0, 4.0) + +prob = ODEProblem(lotka_volterra, u0, tspan, p) + +# Solve using OrdinaryDiffEq.jl solver +dt = 0.01 +solution = solve(prob, Tsit5(); saveat = dt) + +times = solution.t +u = hcat(solution.u...) +x = u[1, :] + (0.4 .* randn(length(u[1, :]))) +y = u[2, :] + (0.4 .* randn(length(u[2, :]))) +dataset = [x, y, times] +scatter!(times, x) +scatter!(times, y) +chain = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), + Lux.Dense(6, 2)) + +alg1 = BNNODE(chain; + dataset = dataset, + draw_samples = 1000, + l2std = [0.1, 0.1], + phystd = [0.2, 0.2], + priorsNNw = (0.0, 1.0), + param = [ + Normal(1,1), + Normal(1,1),], progress = true) + +alg2 = BNNODE(chain; + dataset = dataset, + draw_samples = 1000, + l2std = [0.1, 0.1], + phystd = [0.2, 0.2], + phynewstd = [0.2, 0.2], + priorsNNw = (0.0, 1.0), + param = [ + Normal(1,1), + Normal(1,1)], estim_collocate = true, progress = true) + +@time sol_pestim1 = solve(prob, alg1; saveat = dt) +@time sol_pestim2 = solve(prob, alg2; saveat = dt) + +unsafe_comparisons(true) +bitvec = abs.(p .- pmean(sol_pestim1.estimated_de_params)) .> + abs.(p .- pmean(sol_pestim2.estimated_de_params)) +@test bitvec == ones(size(bitvec)) + + +pmean(sol_pestim1.estimated_de_params) + +sol_pestim2.estimated_de_params +sol_pestim1.estimated_de_params +sol_pestim2.estimated_de_params + +sol_pestim1.estimated_de_params +sol_pestim2.estimated_de_params + +sol_pestim1.estimated_de_params +sol_pestim2.estimated_de_params + +p +sol_pestim1.timepoints +plot!(sol_pestim1.timepoints, sol_pestim1.ensemblesol[1]) +plot!(sol_pestim2.timepoints, sol_pestim2.ensemblesol[1]) +plot!(sol_pestim1.timepoints, sol_pestim1.ensemblesol[2]) +plot!(sol_pestim2.timepoints, sol_pestim2.ensemblesol[2]) + +plot!(sol_pestim1.timepoints, pmean(sol_pestim1.ensemblesol[1])) +plot!(sol_pestim2.timepoints, pmean(sol_pestim2.ensemblesol[1])) +plot!(sol_pestim1.timepoints, pmean(sol_pestim1.ensemblesol[2])) +plot!(sol_pestim2.timepoints, pmean(sol_pestim2.ensemblesol[2])) + + +plot(times, u[1, :]) +plot!(times, u[2, :]) + +plot(sol_pestim1.ensemblesol) + +# Parametric PDEs are ill posed problems as, non convex optimization and non global minima might be our solution \ No newline at end of file From 60b1351a1dff8f6460e6a6075139200e1bb62fe1 Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Fri, 18 Oct 2024 20:32:57 +0530 Subject: [PATCH 52/53] conflicts --- Project.toml | 103 +++++++++++++++-------- src/BPINN_ode.jl | 214 ++++++++++++++++++----------------------------- 2 files changed, 149 insertions(+), 168 deletions(-) diff --git a/Project.toml b/Project.toml index 32d95d792b..d0ddd63ccf 100644 --- a/Project.toml +++ b/Project.toml @@ -1,100 +1,133 @@ name = "NeuralPDE" uuid = "315f7962-48a3-4962-8226-d0f33b1235f0" authors = ["Chris Rackauckas "] -version = "5.16.0" +version = "5.17.0" [deps] +ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" AdvancedHMC = "0bf59076-c3b1-5ca4-86bd-e02cd72cde3d" ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" +ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471" Cubature = "667455a9-e2ce-5579-9412-b964f529a492" -DiffEqNoiseProcess = "77a26b50-5914-5dd7-bc55-306e6241c503" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" DomainSets = "5b8099bc-c8ec-5219-889f-1d9e522a28bf" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" Functors = "d9f16b24-f501-4c13-a1f2-28368ffc5196" Integrals = "de52edbc-65ea-441a-8357-d3a637375a31" +IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" LogDensityProblems = "6fdf6af0-433a-55f7-b3ed-c6c6e0b8df7c" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" +LuxCore = "bb33d45b-7691-41d6-9220-0943567d0623" MCMCChains = "c7f686f2-ff18-58e9-bc7b-31028e88f75d" +MLDataDevices = "7e8f7934-dd98-4c1a-8fe8-92b47a384d40" ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" MonteCarloMeasurements = "0987c9cc-fe09-11e8-30f0-b96dd679fdca" -Optim = "429524aa-4258-5aef-a3af-852621145aeb" +Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" OptimizationOptimisers = "42dfb2eb-d2b4-4451-abcd-913932933ac1" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" QuasiMonteCarlo = "8a4e6c94-4038-4cdc-81c3-7e6ffdb2a71b" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" RuntimeGeneratedFunctions = "7e49a35a-f44a-4d26-94aa-eba1b4ca6b47" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +SymbolicIndexingInterface = "2efcf032-c050-4f8e-a9bb-153293bab1f5" SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b" Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" -UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" +WeightInitializers = "d49dbf32-c5c2-4618-8acc-27bb2598ef2d" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" +[weakdeps] +TensorBoardLogger = "899adc3e-224a-11e9-021f-63837185c80f" + +[extensions] +NeuralPDETensorBoardLoggerExt = "TensorBoardLogger" + [compat] +ADTypes = "1.9.0" Adapt = "4" AdvancedHMC = "0.6.1" -Aqua = "0.8" -ArrayInterface = "7.9" -CUDA = "5.3" +Aqua = "0.8.9" +ArrayInterface = "7.11" +CUDA = "5.5.2" ChainRulesCore = "1.24" -ComponentArrays = "0.15.14" +ComponentArrays = "0.15.16" +ConcreteStructs = "0.2.3" Cubature = "1.5" DiffEqNoiseProcess = "5.20" Distributions = "0.25.107" DocStringExtensions = "0.9.3" -DomainSets = "0.6, 0.7" -Flux = "0.14.11" +DomainSets = "0.7" +ExplicitImports = "1.10.1" +Flux = "0.14.22" ForwardDiff = "0.10.36" -Functors = "0.4.10" -Integrals = "4.4" -LineSearches = "7.2" -LinearAlgebra = "1" +Functors = "0.4.12" +Hwloc = "3.3.0" +Integrals = "4.5" +InteractiveUtils = "<0.0.1, 1" +IntervalSets = "0.7.10" +LineSearches = "7.3" +LinearAlgebra = "1.10" LogDensityProblems = "2" -Lux = "0.5.58" -LuxCUDA = "0.3.2" +Lux = "1.1.0" +LuxCUDA = "0.3.3" +LuxCore = "1.0.1" +LuxLib = "1.3.2" MCMCChains = "6" -MethodOfLines = "0.11" -ModelingToolkit = "9.9" +MLDataDevices = "1.2.0" +MethodOfLines = "0.11.6" +ModelingToolkit = "9.46" MonteCarloMeasurements = "1.1" -Optim = "1.7.8" -Optimization = "3.24, 4" -OptimizationOptimJL = "0.2.1" -OptimizationOptimisers = "0.2.1, 0.3" -OrdinaryDiffEq = "6.74" -Pkg = "1" +Optimisers = "0.3.3" +Optimization = "4" +OptimizationOptimJL = "0.4" +OptimizationOptimisers = "0.3" +OrdinaryDiffEq = "6.87" +Printf = "1.10" QuasiMonteCarlo = "0.3.2" Random = "1" +ReTestItems = "1.29.0" +RecursiveArrayTools = "3.27.0" Reexport = "1.2" RuntimeGeneratedFunctions = "0.5.12" -SafeTestsets = "0.1" -SciMLBase = "2.28" -Statistics = "1.11" -SymbolicUtils = "1.5, 2, 3" -Symbolics = "5.27.1, 6" -Test = "1" -UnPack = "1" -Zygote = "0.6.69" +SciMLBase = "2.56" +Statistics = "1.10" +StochasticDiffEq = "6.69.1" +SymbolicIndexingInterface = "0.3.31" +SymbolicUtils = "3.7.2" +Symbolics = "6.14" +TensorBoardLogger = "0.1.24" +Test = "1.10" +WeightInitializers = "1.0.3" +Zygote = "0.6.71" julia = "1.10" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +DiffEqNoiseProcess = "77a26b50-5914-5dd7-bc55-306e6241c503" +ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +Hwloc = "0e44f5e4-bd66-52a0-8798-143a42290a1d" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" LuxCUDA = "d0bbae9a-e099-4d5b-a835-1c6931763bda" +LuxCore = "bb33d45b-7691-41d6-9220-0943567d0623" +LuxLib = "82251201-b29d-42c6-8e01-566dec8acb11" MethodOfLines = "94925ecb-adb7-4558-8ed8-f975c56a0bf4" OptimizationOptimJL = "36348300-93cb-4f02-beb5-3c3902f8871e" OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" +ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823" +StochasticDiffEq = "789caeaf-c7a9-5a7d-9973-96adeb23e2a0" +TensorBoardLogger = "899adc3e-224a-11e9-021f-63837185c80f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Test", "CUDA", "SafeTestsets", "OptimizationOptimJL", "Pkg", "OrdinaryDiffEq", "LineSearches", "LuxCUDA", "Flux", "MethodOfLines"] +test = ["Aqua", "CUDA", "DiffEqNoiseProcess", "ExplicitImports", "Flux", "Hwloc", "InteractiveUtils", "LineSearches", "LuxCUDA", "LuxCore", "LuxLib", "MethodOfLines", "OptimizationOptimJL", "OrdinaryDiffEq", "ReTestItems", "StochasticDiffEq", "TensorBoardLogger", "Test"] \ No newline at end of file diff --git a/src/BPINN_ode.jl b/src/BPINN_ode.jl index 39bb0aac72..243d681298 100644 --- a/src/BPINN_ode.jl +++ b/src/BPINN_ode.jl @@ -1,16 +1,18 @@ # HIGH level API for BPINN ODE solver """ - BNNODE(chain, Kernel = HMC; strategy = nothing, draw_samples = 2000, - priorsNNw = (0.0, 2.0), param = [nothing], l2std = [0.05], - phystd = [0.05], dataset = [nothing], physdt = 1 / 20.0, - MCMCargs = (n_leapfrog=30), nchains = 1, init_params = nothing, - Adaptorkwargs = (Adaptor = StanHMCAdaptor, targetacceptancerate = 0.8, Metric = DiagEuclideanMetric), - Integratorkwargs = (Integrator = Leapfrog,), autodiff = false, - progress = false, verbose = false) - -Algorithm for solving ordinary differential equations using a Bayesian neural network. This is a specialization -of the physics-informed neural network which is used as a solver for a standard `ODEProblem`. + BNNODE(chain, kernel = HMC; strategy = nothing, draw_samples = 2000, + priorsNNw = (0.0, 2.0), param = [nothing], l2std = [0.05], + phystd = [0.05], dataset = [nothing], physdt = 1 / 20.0, + MCMCargs = (; n_leapfrog=30), nchains = 1, init_params = nothing, + Adaptorkwargs = (; Adaptor = StanHMCAdaptor, targetacceptancerate = 0.8, + Metric = DiagEuclideanMetric), + Integratorkwargs = (Integrator = Leapfrog,), autodiff = false, + progress = false, verbose = false) + +Algorithm for solving ordinary differential equations using a Bayesian neural network. This +is a specialization of the physics-informed neural network which is used as a solver for a +standard `ODEProblem`. !!! warn @@ -20,10 +22,11 @@ of the physics-informed neural network which is used as a solver for a standard ## Positional Arguments -* `chain`: A neural network architecture, defined as a `Lux.AbstractExplicitLayer`. -* `Kernel`: Choice of MCMC Sampling Algorithm. Defaults to `AdvancedHMC.HMC` +* `chain`: A neural network architecture, defined as a `Lux.AbstractLuxLayer`. +* `kernel`: Choice of MCMC Sampling Algorithm. Defaults to `AdvancedHMC.HMC` ## Keyword Arguments + (refer `NeuralPDE.ahmc_bayesian_pinn_ode` keyword arguments.) ## Example @@ -44,18 +47,15 @@ dataset = [x̂, time] chainlux = Lux.Chain(Lux.Dense(1, 6, tanh), Lux.Dense(6, 6, tanh), Lux.Dense(6, 1)) -alg = BNNODE(chainlux, draw_samples = 2000, - l2std = [0.05], phystd = [0.05], - priorsNNw = (0.0, 3.0), progress = true) +alg = BNNODE(chainlux; draw_samples = 2000, l2std = [0.05], phystd = [0.05], + priorsNNw = (0.0, 3.0), progress = true) sol_lux = solve(prob, alg) # with parameter estimation -alg = BNNODE(chainlux,dataset = dataset, - draw_samples = 2000,l2std = [0.05], - phystd = [0.05],priorsNNw = (0.0, 10.0), - param = [Normal(6.5, 0.5), Normal(-3, 0.5)], - progress = true) +alg = BNNODE(chainlux; dataset, draw_samples = 2000, l2std = [0.05], phystd = [0.05], + priorsNNw = (0.0, 10.0), param = [Normal(6.5, 0.5), Normal(-3, 0.5)], + progress = true) sol_lux_pestim = solve(prob, alg) ``` @@ -71,62 +71,49 @@ is an accurate interpolation (up to the neural network training result). In addi ## References -Liu Yanga, Xuhui Menga, George Em Karniadakis. "B-PINNs: Bayesian Physics-Informed Neural Networks for -Forward and Inverse PDE Problems with Noisy Data". +Liu Yanga, Xuhui Menga, George Em Karniadakis. "B-PINNs: Bayesian Physics-Informed Neural +Networks for Forward and Inverse PDE Problems with Noisy Data". Kevin Linka, Amelie Schäfer, Xuhui Meng, Zongren Zou, George Em Karniadakis, Ellen Kuhl "Bayesian Physics Informed Neural Networks for real-world nonlinear dynamical systems". """ -struct BNNODE{C, K, IT <: NamedTuple, - A <: NamedTuple, H <: NamedTuple, - ST <: Union{Nothing, AbstractTrainingStrategy}, - I <: Union{Nothing, <:NamedTuple, Vector{<:AbstractFloat}}, - P <: Union{Nothing, Vector{<:Distribution}}, - D <: - Union{Vector{Nothing}, Vector{<:Vector{<:AbstractFloat}}}} <: - NeuralPDEAlgorithm - chain::C - Kernel::K - strategy::ST - draw_samples::Int64 +@concrete struct BNNODE <: NeuralPDEAlgorithm + chain <: AbstractLuxLayer + kernel + strategy <: Union{Nothing, AbstractTrainingStrategy} + draw_samples::Int priorsNNw::Tuple{Float64, Float64} - param::P + param <: Union{Nothing, Vector{<:Distribution}} l2std::Vector{Float64} phystd::Vector{Float64} phynewstd::Vector{Float64} - dataset::D + dataset <: Union{Vector{Nothing}, Vector{<:Vector{<:AbstractFloat}}} physdt::Float64 - MCMCkwargs::H - nchains::Int64 - init_params::I - Adaptorkwargs::A - Integratorkwargs::IT - numensemble::Int64 + MCMCkwargs <: NamedTuple + nchains::Int + init_params <: Union{Nothing, <:NamedTuple, Vector{<:AbstractFloat}} + Adaptorkwargs <: NamedTuple + Integratorkwargs <: NamedTuple + numensemble::Int estim_collocate::Bool autodiff::Bool progress::Bool verbose::Bool end -function BNNODE(chain, Kernel = HMC; strategy = nothing, draw_samples = 2000, - priorsNNw = (0.0, 2.0), param = nothing, l2std = [0.05], phystd = [0.05], phynewstd = [0.05], - dataset = [nothing], physdt = 1 / 20.0, MCMCkwargs = (n_leapfrog = 30,), nchains = 1, - init_params = nothing, + +function BNNODE(chain, kernel = HMC; strategy = nothing, draw_samples = 2000, + priorsNNw = (0.0, 2.0), param = nothing, l2std = [0.05], phystd = [0.05], + phynewstd = [0.05], dataset = [nothing], physdt = 1 / 20.0, + MCMCkwargs = (n_leapfrog = 30,), nchains = 1, init_params = nothing, Adaptorkwargs = (Adaptor = StanHMCAdaptor, - Metric = DiagEuclideanMetric, - targetacceptancerate = 0.8), + Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), Integratorkwargs = (Integrator = Leapfrog,), numensemble = floor(Int, draw_samples / 3), - estim_collocate = false, - autodiff = false, progress = false, verbose = false) - !(chain isa Lux.AbstractExplicitLayer) && - (chain = adapt(FromFluxAdaptor(false, false), chain)) - BNNODE(chain, Kernel, strategy, - draw_samples, priorsNNw, param, l2std, - phystd, phynewstd, dataset, physdt, MCMCkwargs, - nchains, init_params, - Adaptorkwargs, Integratorkwargs, - numensemble, estim_collocate, - autodiff, progress, verbose) + estim_collocate = false, autodiff = false, progress = false, verbose = false) + chain isa AbstractLuxLayer || (chain = FromFluxAdaptor()(chain)) + return BNNODE(chain, kernel, strategy, draw_samples, priorsNNw, param, l2std, phystd, + phynewstd, dataset, physdt, MCMCkwargs, nchains, init_params, Adaptorkwargs, + Integratorkwargs, numensemble, estim_collocate, autodiff, progress, verbose) end """ @@ -144,98 +131,59 @@ Contains `ahmc_bayesian_pinn_ode()` function output: - step_size - nom_step_size """ -struct BPINNstats{MC, S, ST} - mcmc_chain::MC - samples::S - statistics::ST +@concrete struct BPINNstats + mcmc_chain + samples + statistics end """ -BPINN Solution contains the original solution from AdvancedHMC.jl sampling (BPINNstats contains fields related to that). +BPINN Solution contains the original solution from AdvancedHMC.jl sampling (BPINNstats +contains fields related to that). -1. `ensemblesol` is the Probabilistic Estimate (MonteCarloMeasurements.jl Particles type) of Ensemble solution from All Neural Network's (made using all sampled parameters) output's. +1. `ensemblesol` is the Probabilistic Estimate (MonteCarloMeasurements.jl Particles type) of + Ensemble solution from All Neural Network's (made using all sampled parameters) output's. 2. `estimated_nn_params` - Probabilistic Estimate of NN params from sampled weights, biases. -3. `estimated_de_params` - Probabilistic Estimate of DE params from sampled unknown DE parameters. +3. `estimated_de_params` - Probabilistic Estimate of DE params from sampled unknown DE + parameters. """ -struct BPINNsolution{O <: BPINNstats, E, NP, OP, P} - original::O - ensemblesol::E - estimated_nn_params::NP - estimated_de_params::OP - timepoints::P - - function BPINNsolution(original, - ensemblesol, - estimated_nn_params, - estimated_de_params, - timepoints) - new{typeof(original), typeof(ensemblesol), typeof(estimated_nn_params), - typeof(estimated_de_params), typeof(timepoints)}( - original, ensemblesol, estimated_nn_params, - estimated_de_params, timepoints) - end +@concrete struct BPINNsolution + original <: BPINNstats + ensemblesol + estimated_nn_params + estimated_de_params + timepoints end -function SciMLBase.__solve(prob::SciMLBase.ODEProblem, - alg::BNNODE, - args...; - dt = nothing, - timeseries_errors = true, - save_everystep = true, - adaptive = false, - abstol = 1.0f-6, - reltol = 1.0f-3, - verbose = false, - saveat = 1 / 50.0, - maxiters = nothing,) - @unpack chain, l2std, phystd, phynewstd, param, priorsNNw, Kernel, strategy, - draw_samples, dataset, init_params, - nchains, physdt, Adaptorkwargs, Integratorkwargs, - MCMCkwargs, numensemble, estim_collocate, autodiff, progress, - verbose = alg +function SciMLBase.__solve(prob::SciMLBase.ODEProblem, alg::BNNODE, args...; dt = nothing, + timeseries_errors = true, save_everystep = true, adaptive = false, + abstol = 1.0f-6, reltol = 1.0f-3, verbose = false, saveat = 1 / 50.0, + maxiters = nothing) + (; chain, param, strategy, draw_samples, numensemble, verbose) = alg # ahmc_bayesian_pinn_ode needs param=[] for easier vcat operation for full vector of parameters param = param === nothing ? [] : param strategy = strategy === nothing ? GridTraining : strategy - if draw_samples < 0 - throw(error("Number of samples to be drawn has to be >=0.")) - end + @assert alg.draw_samples≥0 "Number of samples to be drawn has to be >=0." - mcmcchain, samples, statistics = ahmc_bayesian_pinn_ode(prob, chain, - strategy = strategy, dataset = dataset, - draw_samples = draw_samples, - init_params = init_params, - physdt = physdt, phynewstd = phynewstd, - l2std = l2std, - phystd = phystd, - priorsNNw = priorsNNw, - param = param, - nchains = nchains, - autodiff = autodiff, - Kernel = Kernel, - Adaptorkwargs = Adaptorkwargs, - Integratorkwargs = Integratorkwargs, - MCMCkwargs = MCMCkwargs, - progress = progress, - verbose = verbose, - estim_collocate = estim_collocate) + mcmcchain, samples, statistics = ahmc_bayesian_pinn_ode( + prob, chain; strategy, alg.dataset, alg.draw_samples, alg.init_params, + alg.physdt, alg.l2std, alg.phystd, alg.phynewstd, alg.priorsNNw, param, alg.nchains, + alg.autodiff, Kernel = alg.kernel, alg.Adaptorkwargs, alg.Integratorkwargs, + alg.MCMCkwargs, alg.progress, alg.verbose, alg.estim_collocate) fullsolution = BPINNstats(mcmcchain, samples, statistics) ninv = length(param) t = collect(eltype(saveat), prob.tspan[1]:saveat:prob.tspan[2]) - if chain isa Lux.AbstractExplicitLayer - θinit, st = Lux.setup(Random.default_rng(), chain) - θ = [vector_to_parameters(samples[i][1:(end - ninv)], θinit) - for i in 1:max(draw_samples - draw_samples ÷ 10, draw_samples - 1000)] + θinit, st = LuxCore.setup(Random.default_rng(), chain) + θ = [vector_to_parameters(samples[i][1:(end - ninv)], θinit) + for i in 1:max(draw_samples - draw_samples ÷ 10, draw_samples - 1000)] - luxar = [chain(t', θ[i], st)[1] for i in 1:numensemble] - # only need for size - θinit = collect(ComponentArrays.ComponentArray(θinit)) - else - throw(error("Only Lux.AbstractExplicitLayer neural networks are supported")) - end + luxar = [chain(t', θ[i], st)[1] for i in 1:numensemble] + # only need for size + θinit = collect(ComponentArray(θinit)) # constructing ensemble predictions ensemblecurves = Vector{}[] @@ -278,5 +226,5 @@ function SciMLBase.__solve(prob::SciMLBase.ODEProblem, for i in (nnparams + 1):(nnparams + ninv)] end - BPINNsolution(fullsolution, ensemblecurves, estimnnparams, estimated_params, t) -end + return BPINNsolution(fullsolution, ensemblecurves, estimnnparams, estimated_params, t) +end \ No newline at end of file From f5eca91691131e0f49f409cfb7362d4cab41b0dd Mon Sep 17 00:00:00 2001 From: Astitva Aggarwal Date: Fri, 18 Oct 2024 21:55:11 +0530 Subject: [PATCH 53/53] managing conflicts 2 --- src/PDE_BPINN.jl | 441 ++++++++++++----------------- src/advancedHMC_MCMC.jl | 603 +++++++++++++++------------------------- src/discretize.jl | 434 ++++++++++------------------- 3 files changed, 564 insertions(+), 914 deletions(-) diff --git a/src/PDE_BPINN.jl b/src/PDE_BPINN.jl index 8957df889b..4f7e51b3a0 100644 --- a/src/PDE_BPINN.jl +++ b/src/PDE_BPINN.jl @@ -1,70 +1,27 @@ -mutable struct PDELogTargetDensity{ - ST <: AbstractTrainingStrategy, - D <: Union{Nothing, Vector{<:Matrix{<:Real}}}, - P <: Vector{<:Distribution}, - I, - F, - FF, - PH -} - dim::Int64 - strategy::ST - dataset::D - priors::P +@concrete struct PDELogTargetDensity + dim::Int + strategy <: AbstractTrainingStrategy + dataset <: Union{Nothing, Vector{<:Matrix{<:Real}}} + priors <: Vector{<:Distribution} allstd::Vector{Vector{Float64}} + phynewstd::Vector{Float64} names::Tuple extraparams::Int - init_params::I - full_loglikelihood::F - L2_loss2::FF - Φ::PH - - function PDELogTargetDensity(dim, strategy, dataset, - priors, allstd, names, extraparams, - init_params::AbstractVector, full_loglikelihood, L2_loss2, Φ) - new{ - typeof(strategy), - typeof(dataset), - typeof(priors), - typeof(init_params), - typeof(full_loglikelihood), - typeof(L2_loss2), - typeof(Φ) - }(dim, - strategy, - dataset, - priors, - allstd, - names, - extraparams, - init_params, - full_loglikelihood, - L2_loss2, - Φ) - end - function PDELogTargetDensity(dim, strategy, dataset, - priors, allstd, names, extraparams, - init_params::Union{NamedTuple, ComponentArrays.ComponentVector}, - full_loglikelihood, L2_loss2, Φ) - new{ - typeof(strategy), - typeof(dataset), - typeof(priors), - typeof(init_params), - typeof(full_loglikelihood), - typeof(L2_loss2), - typeof(Φ) - }(dim, - strategy, - dataset, - priors, - allstd, - names, - extraparams, - init_params, - full_loglikelihood, - L2_loss2, - Φ) + init_params <: Union{AbstractVector, NamedTuple, ComponentArray} + full_loglikelihood::Any + L2_loss2::Any + Φ::Any +end + +function LogDensityProblems.logdensity(ltd::PDELogTargetDensity, θ) + # for parameter estimation neccesarry to use multioutput case + if Tar.L2_loss2 === nothing + return Tar.full_loglikelihood(setparameters(Tar, θ), Tar.allstd) + + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) + else + return Tar.full_loglikelihood(setparameters(Tar, θ), Tar.allstd) + + priorlogpdf(Tar, θ) + L2LossData(Tar, θ) + + Tar.L2_loss2(setparameters(Tar, θ), Tar.phynewstd) end end @@ -87,7 +44,7 @@ function get_lossy(pinnrep, dataset, Dict_differentials) # for each dataset point(eq_sub dictionary), substitute in masked equations # n_collocated_equations = n_rows_dataset(or n_indvar_coords_dataset) masked_colloc_equations = [[substitute(eq, eq_sub) for eq in eqs_new] - for eq_sub in eq_subs] + for eq_sub in eq_subs] # now we have vector of dataset depvar's collocated equations # reverse dict for re-substituting values of Differential(t)(u(t)) etc @@ -95,15 +52,17 @@ function get_lossy(pinnrep, dataset, Dict_differentials) # unmask Differential terms in masked_colloc_equations colloc_equations = [substitute.(masked_colloc_equation, Ref(rev_Dict_differentials)) - for masked_colloc_equation in masked_colloc_equations] + for masked_colloc_equation in masked_colloc_equations] # nested vector of datafree_pde_loss_functions (as in discretize.jl) # each sub vector has dataset's indvar coord's datafree_colloc_loss_function, n_subvectors = n_rows_dataset(or n_indvar_coords_dataset) # zip each colloc equation with args for each build_loss call per equation vector datafree_colloc_loss_functions = [[build_loss_function(pinnrep, eq, pde_indvar) - for (eq, pde_indvar, integration_indvar) in zip(colloc_equation, - pinnrep.pde_indvars, - pinnrep.pde_integration_vars)] for colloc_equation in colloc_equations] + for (eq, pde_indvar, integration_indvar) in zip( + colloc_equation, + pinnrep.pde_indvars, + pinnrep.pde_integration_vars)] + for colloc_equation in colloc_equations] return datafree_colloc_loss_functions end @@ -132,22 +91,10 @@ function get_symbols(dataset, depvars, eqs) return to_subs, tobe_subs end -function LogDensityProblems.logdensity(Tar::PDELogTargetDensity, θ) - # for parameter estimation neccesarry to use multioutput case - if Tar.L2_loss2 === nothing - return Tar.full_loglikelihood(setparameters(Tar, θ), Tar.allstd) + - priorlogpdf(Tar, θ) + L2LossData(Tar, θ) - else - return Tar.full_loglikelihood(setparameters(Tar, θ), Tar.allstd) + - priorlogpdf(Tar, θ) + L2LossData(Tar, θ) + - Tar.L2_loss2(setparameters(Tar, θ), Tar.allstd) - end -end - -function setparameters(Tar::PDELogTargetDensity, θ) - names = Tar.names - ps_new = θ[1:(end - Tar.extraparams)] - ps = Tar.init_params +@views function setparameters(ltd::PDELogTargetDensity, θ) + names = ltd.names + ps_new = θ[1:(end - ltd.extraparams)] + ps = ltd.init_params # multioutput case for Lux chains, for each depvar ps would contain Lux ComponentVectors # which we use for mapping current ahmc sampled vector of parameters onto NNs @@ -155,83 +102,89 @@ function setparameters(Tar::PDELogTargetDensity, θ) Luxparams = [vector_to_parameters(ps_new[((i += length(ps[x])) - length(ps[x]) + 1):i], ps[x]) for x in names] - a = ComponentArrays.ComponentArray(NamedTuple{Tar.names}(i for i in Luxparams)) + a = ComponentArray(NamedTuple{ltd.names}(i for i in Luxparams)) - if Tar.extraparams > 0 - b = θ[(end - Tar.extraparams + 1):end] - return ComponentArrays.ComponentArray(; - depvar = a, - p = b) + if ltd.extraparams > 0 + return ComponentArray(; depvar = a, p = θ[(end - ltd.extraparams + 1):end]) else - return ComponentArrays.ComponentArray(; - depvar = a) + return ComponentArray(; depvar = a) end end -LogDensityProblems.dimension(Tar::PDELogTargetDensity) = Tar.dim +LogDensityProblems.dimension(ltd::PDELogTargetDensity) = ltd.dim function LogDensityProblems.capabilities(::PDELogTargetDensity) LogDensityProblems.LogDensityOrder{1}() end # L2 losses loglikelihood(needed mainly for ODE parameter estimation) -function L2LossData(Tar::PDELogTargetDensity, θ) - Φ = Tar.Φ - init_params = Tar.init_params - dataset = Tar.dataset - sumt = 0 - L2stds = Tar.allstd[3] +function L2LossData(ltd::PDELogTargetDensity, θ) + Φ = ltd.Φ + init_params = ltd.init_params + dataset = ltd.dataset + L2stds = ltd.allstd[3] # each dep var has a diff dataset depending on its indep var and their domains # these datasets are matrices of first col-dep var and remaining cols-all indep var - # Tar.init_params is needed to construct a vector of parameters into a ComponentVector + # ltd.init_params is needed to construct a vector of parameters into a ComponentVector # dataset of form Vector[matrix_x, matrix_y, matrix_z] # matrix_i is of form [i,indvar1,indvar2,..] (needed in case if heterogenous domains) - # note that indvar1,indvar2.. cols can be different values for different depvar matrices - # dataset,phi order follows pinnrep.depvars orders of variables (order of declaration in @variables macro) # Phi is the trial solution for each NN in chain array # Creating logpdf( MvNormal(Phi(t,θ),std), dataset[i] ) - # dataset[i][:, 2:end] -> indepvar cols of a particular depvar's dataset + # dataset[i][:, 2:end] -> indepvar cols of a particular depvar's dataset # dataset[i][:, 1] -> depvar col of depvar's dataset - if Tar.extraparams > 0 - for i in eachindex(Φ) - sumt += logpdf( - MvNormal( - Φ[i](dataset[i][:, 2:end]', - vector_to_parameters(θ[1:(end - Tar.extraparams)], - init_params)[Tar.names[i]])[1, - :], - LinearAlgebra.Diagonal(abs2.(ones(size(dataset[i])[1]) .* - L2stds[i]))), - dataset[i][:, 1]) - end - return sumt + ltd.extraparams ≤ 0 && return false + + sumt = 0 + for i in eachindex(Φ) + sumt += logpdf( + MvNormal( + Φ[i](dataset[i][:, 2:end]', + vector_to_parameters(θ[1:(end - ltd.extraparams)], init_params)[ltd.names[i]])[ + 1, :], + Diagonal(abs2.(ones(size(dataset[i])[1]) .* L2stds[i]))), + dataset[i][:, 1]) end - return 0 + return sumt end # priors for NN parameters + ODE constants -function priorlogpdf(Tar::PDELogTargetDensity, θ) - allparams = Tar.priors +function priorlogpdf(ltd::PDELogTargetDensity, θ) + allparams = ltd.priors # Vector of ode parameters priors invpriors = allparams[2:end] - - # nn weights nnwparams = allparams[1] - if Tar.extraparams > 0 - invlogpdf = sum( - logpdf(invpriors[length(θ) - i + 1], θ[i]) - for i in (length(θ) - Tar.extraparams + 1):length(θ); - init = 0.0) + ltd.extraparams ≤ 0 && return logpdf(nnwparams, θ) - return (invlogpdf - + - logpdf(nnwparams, θ[1:(length(θ) - Tar.extraparams)])) + invlogpdf = sum((length(θ) - ltd.extraparams + 1):length(θ)) do i + logpdf(invpriors[length(θ) - i + 1], θ[i]) + end + + return invlogpdf + logpdf(nnwparams, θ[1:(length(θ) - ltd.extraparams)]) +end + +function integratorchoice(Integratorkwargs, initial_ϵ) + Integrator = Integratorkwargs[:Integrator] + if Integrator == JitteredLeapfrog + jitter_rate = Integratorkwargs[:jitter_rate] + Integrator(initial_ϵ, jitter_rate) + elseif Integrator == TemperedLeapfrog + tempering_rate = Integratorkwargs[:tempering_rate] + Integrator(initial_ϵ, tempering_rate) + else + Integrator(initial_ϵ) + end +end + +function adaptorchoice(Adaptor, mma, ssa) + if Adaptor != AdvancedHMC.NoAdaptation() + Adaptor(mma, ssa) + else + AdvancedHMC.NoAdaptation() end - return logpdf(nnwparams, θ) end function inference(samples, pinnrep, saveats, numensemble, ℓπ) @@ -298,82 +251,69 @@ function inference(samples, pinnrep, saveats, numensemble, ℓπ) return ensemblecurves, estimatedLuxparams, estimated_params, timepoints end -function integratorchoice(Integratorkwargs, initial_ϵ) - Integrator = Integratorkwargs[:Integrator] - if Integrator == JitteredLeapfrog - jitter_rate = Integratorkwargs[:jitter_rate] - Integrator(initial_ϵ, jitter_rate) - elseif Integrator == TemperedLeapfrog - tempering_rate = Integratorkwargs[:tempering_rate] - Integrator(initial_ϵ, tempering_rate) - else - Integrator(initial_ϵ) - end -end - -function adaptorchoice(Adaptor, mma, ssa) - if Adaptor != AdvancedHMC.NoAdaptation() - Adaptor(mma, ssa) - else - AdvancedHMC.NoAdaptation() - end -end - """ ahmc_bayesian_pinn_pde(pde_system, discretization; - draw_samples = 1000, - bcstd = [0.01], l2std = [0.05], - phystd = [0.05], priorsNNw = (0.0, 2.0), - param = [], nchains = 1, Kernel = HMC(0.1, 30), - Adaptorkwargs = (Adaptor = StanHMCAdaptor, - Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), - Integratorkwargs = (Integrator = Leapfrog,), saveats = [1 / 10.0], - numensemble = floor(Int, draw_samples / 3), progress = false, verbose = false) + draw_samples = 1000, bcstd = [0.01], l2std = [0.05], phystd = [0.05], + priorsNNw = (0.0, 2.0), param = [], nchains = 1, Kernel = HMC(0.1, 30), + Adaptorkwargs = (Adaptor = StanHMCAdaptor, + Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), + Integratorkwargs = (Integrator = Leapfrog,), saveats = [1 / 10.0], + numensemble = floor(Int, draw_samples / 3), progress = false, verbose = false) ## NOTES * Dataset is required for accurate Parameter estimation + solving equations. -* Returned solution is a BPINNsolution consisting of Ensemble solution, estimated PDE and NN parameters - for chosen `saveats` grid spacing and last n = `numensemble` samples in Chain. the complete set of samples - in the MCMC chain is returned as `fullsolution`, refer `BPINNsolution` for more details. +* Returned solution is a BPINNsolution consisting of Ensemble solution, estimated PDE and NN + parameters for chosen `saveats` grid spacing and last n = `numensemble` samples in Chain. + the complete set of samples in the MCMC chain is returned as `fullsolution`, refer + `BPINNsolution` for more details. ## Positional Arguments * `pde_system`: ModelingToolkit defined PDE equation or system of equations. -* `discretization`: BayesianPINN discretization for the given pde_system, Neural Network and training strategy. +* `discretization`: BayesianPINN discretization for the given pde_system, Neural Network and + training strategy. ## Keyword Arguments -* `draw_samples`: number of samples to be drawn in the MCMC algorithms (warmup samples are ~2/3 of draw samples) -* `bcstd`: Vector of standard deviations of BPINN prediction against Initial/Boundary Condition equations. -* `l2std`: Vector of standard deviations of BPINN prediction against L2 losses/Dataset for each dependant variable of interest. -* `phystd`: Vector of standard deviations of BPINN prediction against Chosen Underlying PDE equations. -* `priorsNNw`: Tuple of (mean, std) for BPINN Network parameters. Weights and Biases of BPINN are Normal Distributions by default. +* `draw_samples`: number of samples to be drawn in the MCMC algorithms (warmup samples are + ~2/3 of draw samples) +* `bcstd`: Vector of standard deviations of BPINN prediction against Initial/Boundary + Condition equations. +* `l2std`: Vector of standard deviations of BPINN prediction against L2 losses/Dataset for + each dependant variable of interest. +* `phystd`: Vector of standard deviations of BPINN prediction against Chosen Underlying PDE + equations. +* `priorsNNw`: Tuple of (mean, std) for BPINN Network parameters. Weights and Biases of + BPINN are Normal Distributions by default. * `param`: Vector of chosen PDE's parameter's Distributions in case of Inverse problems. * `nchains`: number of chains you want to sample. -* `Kernel`: Choice of MCMC Sampling Algorithm object HMC/NUTS/HMCDA (AdvancedHMC.jl implementations). -* `Adaptorkwargs`: `Adaptor`, `Metric`, `targetacceptancerate`. Refer: https://turinglang.org/AdvancedHMC.jl/stable/ - Note: Target percentage(in decimal) of iterations in which the proposals are accepted (0.8 by default). -* `Integratorkwargs`: `Integrator`, `jitter_rate`, `tempering_rate`. Refer: https://turinglang.org/AdvancedHMC.jl/stable/ -* `saveats`: Grid spacing for each independent variable for evaluation of ensemble solution, estimated parameters. -* `numensemble`: Number of last samples to take for creation of ensemble solution, estimated parameters. +* `Kernel`: Choice of MCMC Sampling Algorithm object HMC/NUTS/HMCDA (AdvancedHMC.jl + implementations). +* `Adaptorkwargs`: `Adaptor`, `Metric`, `targetacceptancerate`. Refer: + https://turinglang.org/AdvancedHMC.jl/stable/. Note: Target percentage(in decimal) of + iterations in which the proposals are accepted (0.8 by default). +* `Integratorkwargs`: `Integrator`, `jitter_rate`, `tempering_rate`. Refer: + https://turinglang.org/AdvancedHMC.jl/stable/ +* `saveats`: Grid spacing for each independent variable for evaluation of ensemble solution, + estimated parameters. +* `numensemble`: Number of last samples to take for creation of ensemble solution, estimated + parameters. * `progress`: controls whether to show the progress meter or not. * `verbose`: controls the verbosity. (Sample call args in AHMC). -## Warnings +!!! warning -* AdvancedHMC.jl is still developing convenience structs so might need changes on new releases. + AdvancedHMC.jl is still developing convenience structs so might need changes on new + releases. """ function ahmc_bayesian_pinn_pde(pde_system, discretization; - draw_samples = 1000, - bcstd = [0.01], l2std = [0.05], - phystd = [0.05], phystdnew = [0.05], priorsNNw = (0.0, 2.0), - param = [], nchains = 1, Kernel = HMC(0.1, 30), - Adaptorkwargs = (Adaptor = StanHMCAdaptor, + draw_samples = 1000, bcstd = [0.01], l2std = [0.05], phystd = [0.05], + phynewstd = [0.05], priorsNNw = (0.0, 2.0), param = [], nchains = 1, + Kernel = HMC(0.1, 30), Adaptorkwargs = (Adaptor = StanHMCAdaptor, Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), Integratorkwargs = (Integrator = Leapfrog,), saveats = [1 / 10.0], - numensemble = floor(Int, draw_samples / 3), Dict_differentials = nothing, - progress = false, verbose = false) + numensemble = floor(Int, draw_samples / 3), progress = false, verbose = false) pinnrep = symbolic_discretize(pde_system, discretization) dataset_pde, dataset_bc = discretization.dataset @@ -385,31 +325,31 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; # add case for if parameters present in bcs? train_sets_pde = get_dataset_train_points(pde_system.eqs, - dataset_pde, - pinnrep) - colloc_train_sets = [[hcat(train_sets_pde[i][:, j]...)' for i in eachindex(datafree_colloc_loss_functions[1])] for j in eachindex(datafree_colloc_loss_functions)] + dataset_pde, + pinnrep) + colloc_train_sets = [[hcat(train_sets_pde[i][:, j]...)' + for i in eachindex(datafree_colloc_loss_functions[1])] + for j in eachindex(datafree_colloc_loss_functions)] # for each datafree_colloc_loss_function create loss_functions by passing dataset's indvar coords as train_sets_pde. # placeholder strategy = GridTraining(0.1), datafree_bc_loss_function and train_sets_bc must be nothing # order of indvar coords will be same as corresponding depvar coords values in dataset provided in get_lossy() call. pde_loss_function_points = [merge_strategy_with_loglikelihood_function( - pinnrep, - GridTraining(0.1), - datafree_colloc_loss_functions[i], - nothing; - train_sets_pde = colloc_train_sets[i], - train_sets_bc = nothing)[1] - for i in eachindex(datafree_colloc_loss_functions)] - - function L2_loss2(θ, allstd) - stdpdesnew = allstd[4] - + pinnrep, + GridTraining(0.1), + datafree_colloc_loss_functions[i], + nothing; + train_sets_pde = colloc_train_sets[i], + train_sets_bc = nothing)[1] + for i in eachindex(datafree_colloc_loss_functions)] + + function L2_loss2(θ, phynewstd) # first vector of losses,from tuple -> pde losses, first[1] pde loss - pde_loglikelihoods = [sum([pde_loss_function(θ, stdpdesnew[i]) + pde_loglikelihoods = [sum([pde_loss_function(θ, phynewstd[i]) for (i, pde_loss_function) in enumerate(pde_loss_functions)]) for pde_loss_functions in pde_loss_function_points] - # bc_loglikelihoods = [sum([bc_loss_function(θ, stdpdesnew[i]) for (i, bc_loss_function) in enumerate(pde_loss_function_points[1])]) for pde_loss_function_points in pde_loss_functions] + # bc_loglikelihoods = [sum([bc_loss_function(θ, phynewstd[i]) for (i, bc_loss_function) in enumerate(pde_loss_function_points[1])]) for pde_loss_function_points in pde_loss_functions] # for (j, bc_loss_function) in enumerate(bc_loss_functions)] return sum(pde_loglikelihoods) @@ -432,7 +372,7 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; elseif discretization.param_estim && dataset isa Nothing throw(UndefVarError(:dataset)) elseif discretization.param_estim && length(l2std) != length(pinnrep.depvars) - throw(error("L2 stds length must match number of dependant variables")) + error("L2 stds length must match number of dependant variables") end # for physics loglikelihood @@ -440,15 +380,13 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; chain = discretization.chain if length(pinnrep.domains) != length(saveats) - throw(error("Number of independent variables must match saveat inference discretization steps")) + error("Number of independent variables must match saveat inference discretization steps") end # NN solutions for loglikelihood which is used for L2lossdata Φ = pinnrep.phi - if nchains < 1 - throw(error("number of chains must be greater than or equal to 1")) - end + @assert nchains≥1 "number of chains must be greater than or equal to 1" # remove inv params take only NN params, AHMC uses Float64 initial_nnθ = pinnrep.flat_init_params[1:(end - length(param))] @@ -465,7 +403,7 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; # add init_params for NN params priors = [ MvNormal(priorsNNw[1] * ones(nparameters), - LinearAlgebra.Diagonal(abs2.(priorsNNw[2] .* ones(nparameters)))) + Diagonal(abs2.(priorsNNw[2] .* ones(nparameters)))) ] # append Ode params to all paramvector - initial_θ @@ -480,18 +418,10 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; # vector in case of N-dimensional domains strategy = discretization.strategy - # dimensions would be total no of params,initial_nnθ for Lux namedTuples - ℓπ = PDELogTargetDensity(nparameters, - strategy, - dataset, - priors, - [phystd, bcstd, l2std, phystdnew], - names, - ninv, - initial_nnθ, - full_weighted_loglikelihood, - newloss, - Φ) + # dimensions would be total no of params,initial_nnθ for Lux namedTuples + ℓπ = PDELogTargetDensity( + nparameters, strategy, dataset, priors, [phystd, bcstd, l2std], phynewstd, + names, ninv, initial_nnθ, full_weighted_loglikelihood, newloss, Φ) Adaptor, Metric, targetacceptancerate = Adaptorkwargs[:Adaptor], Adaptorkwargs[:Metric], Adaptorkwargs[:targetacceptancerate] @@ -500,19 +430,22 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; metric = Metric(nparameters) hamiltonian = Hamiltonian(metric, ℓπ, ForwardDiff) - @info("Current Physics Log-likelihood : ", - ℓπ.full_loglikelihood(setparameters(ℓπ, initial_θ), - ℓπ.allstd)) - @info("Current Prior Log-likelihood : ", priorlogpdf(ℓπ, initial_θ)) - @info("Current MSE against dataset Log-likelihood : ", L2LossData(ℓπ, initial_θ)) - if !(newloss isa Nothing) - @info("Current L2_LOSSY : ", - ℓπ.L2_loss2(setparameters(ℓπ, initial_θ), - ℓπ.allstd)) + if verbose + @printf("Current Physics Log-likelihood : %g\n", + ℓπ.full_loglikelihood(setparameters(ℓπ, initial_θ), ℓπ.allstd)) + @printf("Current Prior Log-likelihood : %g\n", priorlogpdf(ℓπ, initial_θ)) + @printf("Current MSE against dataset Log-likelihood : %g\n", + L2LossData(ℓπ, initial_θ)) + if !(newloss isa Nothing) + @printf("Current new loss : %g\n", + ℓπ.L2_loss2(setparameters(ℓπ, initial_θ), + ℓπ.phynewstd)) + end end # parallel sampling option if nchains != 1 + # Cache to store the chains bpinnsols = Vector{Any}(undef, nchains) @@ -534,17 +467,10 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; fullsolution = BPINNstats(mcmc_chain, samples, stats) ensemblecurves, estimnnparams, estimated_params, timepoints = inference( - samples, - pinnrep, - saveat, - numensemble, - ℓπ) - - bpinnsols[i] = BPINNsolution(fullsolution, - ensemblecurves, - estimnnparams, - estimated_params, - timepoints) + samples, pinnrep, saveat, numensemble, ℓπ) + + bpinnsols[i] = BPINNsolution( + fullsolution, ensemblecurves, estimnnparams, estimated_params, timepoints) end return bpinnsols else @@ -561,30 +487,25 @@ function ahmc_bayesian_pinn_pde(pde_system, discretization; matrix_samples = hcat(samples...) mcmc_chain = MCMCChains.Chains(matrix_samples') - @info("Sampling Complete.") - @info("Final Physics Log-likelihood : ", - ℓπ.full_loglikelihood(setparameters(ℓπ, samples[end]), - ℓπ.allstd)) - @info("Final Prior Log-likelihood : ", priorlogpdf(ℓπ, samples[end])) - @info("Final MSE against dataset Log-likelihood : ", - L2LossData(ℓπ, samples[end])) - if !(newloss isa Nothing) - @info("Final L2_LOSSY : ", - ℓπ.L2_loss2(setparameters(ℓπ, samples[end]), - ℓπ.allstd)) + if verbose + @printf("Sampling Complete.\n") + @printf("Final Physics Log-likelihood : %g\n", + ℓπ.full_loglikelihood(setparameters(ℓπ, samples[end]), ℓπ.allstd)) + @printf("Final Prior Log-likelihood : %g\n", priorlogpdf(ℓπ, samples[end])) + @printf("Final MSE against dataset Log-likelihood : %g\n", + L2LossData(ℓπ, samples[end])) + if !(newloss isa Nothing) + @printf("Final L2_LOSSY : %g\n", + ℓπ.L2_loss2(setparameters(ℓπ, samples[end]), + ℓπ.phynewstd)) + end end fullsolution = BPINNstats(mcmc_chain, samples, stats) ensemblecurves, estimnnparams, estimated_params, timepoints = inference(samples, - pinnrep, - saveats, - numensemble, - ℓπ) - - return BPINNsolution(fullsolution, - ensemblecurves, - estimnnparams, - estimated_params, - timepoints) + pinnrep, saveats, numensemble, ℓπ) + + return BPINNsolution( + fullsolution, ensemblecurves, estimnnparams, estimated_params, timepoints) end -end +end \ No newline at end of file diff --git a/src/advancedHMC_MCMC.jl b/src/advancedHMC_MCMC.jl index 8b996fce5c..5ac4213c92 100644 --- a/src/advancedHMC_MCMC.jl +++ b/src/advancedHMC_MCMC.jl @@ -1,72 +1,42 @@ -mutable struct LogTargetDensity{C, S, ST <: AbstractTrainingStrategy, I, - P <: Vector{<:Distribution}, - D <: - Union{Vector{Nothing}, Vector{<:Vector{<:AbstractFloat}}} -} +@concrete struct LogTargetDensity dim::Int - prob::SciMLBase.ODEProblem - chain::C - st::S - strategy::ST - dataset::D - priors::P + prob <: SciMLBase.ODEProblem + smodel <: StatefulLuxLayer + strategy <: AbstractTrainingStrategy + dataset <: Union{Vector{Nothing}, Vector{<:Vector{<:AbstractFloat}}} + priors <: Vector{<:Distribution} phystd::Vector{Float64} phynewstd::Vector{Float64} l2std::Vector{Float64} autodiff::Bool physdt::Float64 extraparams::Int - init_params::I + init_params <: Union{NamedTuple, ComponentArray} estim_collocate::Bool +end - function LogTargetDensity(dim, prob, chain::Optimisers.Restructure, st, strategy, - dataset, - priors, phystd, phynewstd, l2std, autodiff, physdt, extraparams, - init_params::AbstractVector, estim_collocate) - new{ - typeof(chain), - Nothing, - typeof(strategy), - typeof(init_params), - typeof(priors), - typeof(dataset) - }(dim, - prob, - chain, - nothing, strategy, - dataset, - priors, - phystd, - phynewstd, - l2std, - autodiff, - physdt, - extraparams, - init_params, - estim_collocate) - end - function LogTargetDensity(dim, prob, chain::Lux.AbstractExplicitLayer, st, strategy, - dataset, - priors, phystd, phynewstd, l2std, autodiff, physdt, extraparams, - init_params::NamedTuple, estim_collocate) - new{ - typeof(chain), - typeof(st), - typeof(strategy), - typeof(init_params), - typeof(priors), - typeof(dataset) - }(dim, - prob, - chain, st, strategy, - dataset, priors, - phystd, phynewstd, - l2std, - autodiff, - physdt, - extraparams, - init_params, - estim_collocate) +""" +NN OUTPUT AT t,θ ~ phi(t,θ). +""" +function (f::LogTargetDensity)(t::AbstractVector, θ) + θ = vector_to_parameters(θ, f.init_params) + dev = safe_get_device(θ) + t = safe_expand(dev, t) + u0 = f.prob.u0 |> dev + return u0 .+ (t' .- f.prob.tspan[1]) .* f.smodel(t', θ) +end + +(f::LogTargetDensity)(t::Number, θ) = f([t], θ)[:, 1] + +""" +Similar to ode_dfdx() in NNODE. +""" +function ode_dfdx(phi::LogTargetDensity, t::AbstractVector, θ, autodiff::Bool) + if autodiff + return ForwardDiff.jacobian(Base.Fix2(phi, θ), t) + else + ϵ = sqrt(eps(eltype(t))) + return (phi(t .+ ϵ, θ) .- phi(t, θ)) ./ ϵ end end @@ -74,344 +44,239 @@ end Function needed for converting vector of sampled parameters into ComponentVector in case of Lux chain output, derivatives the sampled parameters are of exotic type `Dual` due to ForwardDiff's autodiff tagging. """ -function vector_to_parameters(ps_new::AbstractVector, - ps::Union{NamedTuple, ComponentArrays.ComponentVector}) - @assert length(ps_new) == Lux.parameterlength(ps) +function vector_to_parameters(ps_new::AbstractVector, ps::Union{NamedTuple, ComponentArray}) + @assert length(ps_new) == LuxCore.parameterlength(ps) i = 1 function get_ps(x) z = reshape(view(ps_new, i:(i + length(x) - 1)), size(x)) i += length(x) return z end - return Functors.fmap(get_ps, ps) + return fmap(get_ps, ps) end -vector_to_parameters(ps_new::AbstractVector, ps::AbstractVector) = ps_new +vector_to_parameters(ps_new::AbstractVector, _::AbstractVector) = ps_new -function LogDensityProblems.logdensity(Tar::LogTargetDensity, θ) - if Tar.estim_collocate - return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) + - L2loss2(Tar, θ) - else - return physloglikelihood(Tar, θ) + priorweights(Tar, θ) + L2LossData(Tar, θ) - end +function LogDensityProblems.logdensity(ltd::LogTargetDensity, θ) + ldensity = physloglikelihood(ltd, θ) + priorweights(ltd, θ) + L2LossData(ltd, θ) + ltd.estim_collocate && return ldensity + L2loss2(ltd, θ) + return ldensity end -LogDensityProblems.dimension(Tar::LogTargetDensity) = Tar.dim +LogDensityProblems.dimension(ltd::LogTargetDensity) = ltd.dim function LogDensityProblems.capabilities(::LogTargetDensity) - LogDensityProblems.LogDensityOrder{1}() + return LogDensityProblems.LogDensityOrder{1}() end """ suggested extra loss function for ODE solver case """ -function L2loss2(Tar::LogTargetDensity, θ) - f = Tar.prob.f +@views function L2loss2(ltd::LogTargetDensity, θ) + ltd.extraparams ≤ 0 && return false # XXX: type-stability? - # parameter estimation chosen or not - if Tar.extraparams > 0 - autodiff = Tar.autodiff - # Timepoints to enforce Physics - t = Tar.dataset[end] - u1 = Tar.dataset[2] - û = Tar.dataset[1] - - nnsol = NNodederi(Tar, t, θ[1:(length(θ) - Tar.extraparams)], autodiff) - - ode_params = Tar.extraparams == 1 ? - θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : - θ[((length(θ) - Tar.extraparams) + 1):length(θ)] - - if length(Tar.prob.u0) == 1 - physsol = [f(û[i], - ode_params, - t[i]) - for i in 1:length(û[:, 1])] - else - physsol = [f([û[i], u1[i]], - ode_params, - t[i]) - for i in 1:length(û)] - end - #form of NN output matrix output dim x n - deri_physsol = reduce(hcat, physsol) - - physlogprob = 0 - for i in 1:length(Tar.prob.u0) - # can add phystdnew[i] for u[i] - physlogprob += logpdf(MvNormal(deri_physsol[i, :], - LinearAlgebra.Diagonal(map(abs2, - (Tar.phynewstd[i]) .* - ones(length(nnsol[i, :]))))), - nnsol[i, :]) - end - return physlogprob + f = ltd.prob.f + t = ltd.dataset[end] + u1 = ltd.dataset[2] + û = ltd.dataset[1] + + nnsol = ode_dfdx(ltd, t, θ[1:(length(θ) - ltd.extraparams)], ltd.autodiff) + + ode_params = ltd.extraparams == 1 ? θ[((length(θ) - ltd.extraparams) + 1)] : + θ[((length(θ) - ltd.extraparams) + 1):length(θ)] + + physsol = if length(ltd.prob.u0) == 1 + [f(û[i], ode_params, tᵢ) for (i, tᵢ) in enumerate(t)] else - return 0 + [f([û[i], u1[i]], ode_params, tᵢ) for (i, tᵢ) in enumerate(t)] + end + # form of NN output matrix output dim x n + deri_physsol = reduce(hcat, physsol) + T = promote_type(eltype(deri_physsol), eltype(nnsol)) + + physlogprob = T(0) + for i in 1:length(ltd.prob.u0) + physlogprob += logpdf( + MvNormal(deri_physsol[i, :], + Diagonal(abs2.(T(ltd.phynewstd[i]) .* ones(T, length(nnsol[i, :]))))), + nnsol[i, :] + ) end + return physlogprob end """ L2 loss loglikelihood(needed for ODE parameter estimation). """ -function L2LossData(Tar::LogTargetDensity, θ) - # check if dataset is provided - if Tar.dataset isa Vector{Nothing} || Tar.extraparams == 0 - return 0 - else - # matrix(each row corresponds to vector u's rows) - nn = Tar(Tar.dataset[end], θ[1:(length(θ) - Tar.extraparams)]) - - L2logprob = 0 - for i in 1:length(Tar.prob.u0) - # for u[i] ith vector must be added to dataset, nn[1,:] is the dx in lotka_volterra - L2logprob += logpdf( - MvNormal(nn[i, :], - LinearAlgebra.Diagonal(abs2.(Tar.l2std[i] .* - ones(length(Tar.dataset[i]))))), - Tar.dataset[i]) - end - return L2logprob +@views function L2LossData(ltd::LogTargetDensity, θ) + (ltd.dataset isa Vector{Nothing} || ltd.extraparams == 0) && return 0 + + # matrix(each row corresponds to vector u's rows) + nn = ltd(ltd.dataset[end], θ[1:(length(θ) - ltd.extraparams)]) + T = eltype(nn) + + L2logprob = zero(T) + for i in 1:length(ltd.prob.u0) + # for u[i] ith vector must be added to dataset,nn[1, :] is the dx in lotka_volterra + L2logprob += logpdf( + MvNormal( + nn[i, :], + Diagonal(abs2.(T(ltd.l2std[i]) .* ones(T, length(ltd.dataset[i])))) + ), + ltd.dataset[i] + ) end + return L2logprob end """ Physics loglikelihood over problem timespan + dataset timepoints. """ -function physloglikelihood(Tar::LogTargetDensity, θ) - f = Tar.prob.f - p = Tar.prob.p - tspan = Tar.prob.tspan - autodiff = Tar.autodiff - strategy = Tar.strategy +function physloglikelihood(ltd::LogTargetDensity, θ) + (; f, p, tspan) = ltd.prob + (; autodiff, strategy) = ltd # parameter estimation chosen or not - if Tar.extraparams > 0 - ode_params = Tar.extraparams == 1 ? - θ[((length(θ) - Tar.extraparams) + 1):length(θ)][1] : - θ[((length(θ) - Tar.extraparams) + 1):length(θ)] + if ltd.extraparams > 0 + ode_params = ltd.extraparams == 1 ? θ[((length(θ) - ltd.extraparams) + 1)] : + θ[((length(θ) - ltd.extraparams) + 1):length(θ)] else - ode_params = p == SciMLBase.NullParameters() ? [] : p + ode_params = p isa SciMLBase.NullParameters ? Float64[] : p end - return getlogpdf(strategy, Tar, f, autodiff, tspan, ode_params, θ) + return getlogpdf(strategy, ltd, f, autodiff, tspan, ode_params, θ) end -function getlogpdf(strategy::GridTraining, Tar::LogTargetDensity, f, autodiff::Bool, - tspan, - ode_params, θ) - if Tar.dataset isa Vector{Nothing} - t = collect(eltype(strategy.dx), tspan[1]:(strategy.dx):tspan[2]) - else - t = vcat(collect(eltype(strategy.dx), tspan[1]:(strategy.dx):tspan[2]), - Tar.dataset[end]) - end - - sum(innerdiff(Tar, f, autodiff, t, θ, - ode_params)) +function getlogpdf(strategy::GridTraining, ltd::LogTargetDensity, f, autodiff::Bool, + tspan, ode_params, θ) + ts = collect(eltype(strategy.dx), tspan[1]:(strategy.dx):tspan[2]) + t = ltd.dataset isa Vector{Nothing} ? ts : vcat(ts, ltd.dataset[end]) + return sum(innerdiff(ltd, f, autodiff, t, θ, ode_params)) end -function getlogpdf(strategy::StochasticTraining, - Tar::LogTargetDensity, - f, - autodiff::Bool, - tspan, - ode_params, - θ) - if Tar.dataset isa Vector{Nothing} - t = [(tspan[2] - tspan[1]) * rand() + tspan[1] for i in 1:(strategy.points)] - else - t = vcat([(tspan[2] - tspan[1]) * rand() + tspan[1] for i in 1:(strategy.points)], - Tar.dataset[end]) - end - - sum(innerdiff(Tar, f, autodiff, t, θ, - ode_params)) +function getlogpdf(strategy::StochasticTraining, ltd::LogTargetDensity, + f, autodiff::Bool, tspan, ode_params, θ) + T = promote_type(eltype(tspan[1]), eltype(tspan[2])) + samples = (tspan[2] - tspan[1]) .* rand(T, strategy.points) .+ tspan[1] + t = ltd.dataset isa Vector{Nothing} ? samples : vcat(samples, ltd.dataset[end]) + return sum(innerdiff(ltd, f, autodiff, t, θ, ode_params)) end -function getlogpdf(strategy::QuadratureTraining, Tar::LogTargetDensity, f, - autodiff::Bool, - tspan, - ode_params, θ) - function integrand(t::Number, θ) - innerdiff(Tar, f, autodiff, [t], θ, ode_params) - end +function getlogpdf(strategy::QuadratureTraining, ltd::LogTargetDensity, f, autodiff::Bool, + tspan, ode_params, θ) + integrand(t::Number, θ) = innerdiff(ltd, f, autodiff, [t], θ, ode_params) intprob = IntegralProblem( - integrand, (tspan[1], tspan[2]), θ; nout = length(Tar.prob.u0)) - sol = solve(intprob, QuadGKJL(); abstol = strategy.abstol, reltol = strategy.reltol) - sum(sol.u) + integrand, (tspan[1], tspan[2]), θ; nout = length(ltd.prob.u0)) + sol = solve(intprob, QuadGKJL(); strategy.abstol, strategy.reltol) + return sum(sol.u) end -function getlogpdf(strategy::WeightedIntervalTraining, Tar::LogTargetDensity, f, - autodiff::Bool, - tspan, - ode_params, θ) - minT = tspan[1] - maxT = tspan[2] - +function getlogpdf(strategy::WeightedIntervalTraining, ltd::LogTargetDensity, f, + autodiff::Bool, tspan, ode_params, θ) + minT, maxT = tspan weights = strategy.weights ./ sum(strategy.weights) - N = length(weights) - points = strategy.points - difference = (maxT - minT) / N - data = Float64[] + ts = eltype(difference)[] for (index, item) in enumerate(weights) - temp_data = rand(1, trunc(Int, points * item)) .* difference .+ minT .+ + temp_data = rand(1, trunc(Int, strategy.points * item)) .* difference .+ minT .+ ((index - 1) * difference) - data = append!(data, temp_data) + append!(ts, temp_data) end - if Tar.dataset isa Vector{Nothing} - t = data - else - t = vcat(data, - Tar.dataset[end]) - end - - sum(innerdiff(Tar, f, autodiff, t, θ, - ode_params)) + t = ltd.dataset isa Vector{Nothing} ? ts : vcat(ts, ltd.dataset[end]) + return sum(innerdiff(ltd, f, autodiff, t, θ, ode_params)) end """ MvNormal likelihood at each `ti` in time `t` for ODE collocation residue with NN with parameters θ. """ -function innerdiff(Tar::LogTargetDensity, f, autodiff::Bool, t::AbstractVector, θ, +@views function innerdiff(ltd::LogTargetDensity, f, autodiff::Bool, t::AbstractVector, θ, ode_params) + # ltd used for phi and LogTargetDensity object attributes access + out = ltd(t, θ[1:(length(θ) - ltd.extraparams)]) - # Tar used for phi and LogTargetDensity object attributes access - out = Tar(t, θ[1:(length(θ) - Tar.extraparams)]) - - # # reject samples case(write clear reason why) - if any(isinf, out[:, 1]) || any(isinf, ode_params) - return -Inf - end + # reject samples case(write clear reason why) + (any(isinf, out[:, 1]) || any(isinf, ode_params)) && return convert(eltype(out), -Inf) # this is a vector{vector{dx,dy}}(handle case single u(float passed)) if length(out[:, 1]) == 1 - physsol = [f(out[:, i][1], - ode_params, - t[i]) - for i in 1:length(out[1, :])] + physsol = [f(out[:, i][1], ode_params, t[i]) for i in 1:length(out[1, :])] else - physsol = [f(out[:, i], - ode_params, - t[i]) - for i in 1:length(out[1, :])] + physsol = [f(out[:, i], ode_params, t[i]) for i in 1:length(out[1, :])] end physsol = reduce(hcat, physsol) - nnsol = NNodederi(Tar, t, θ[1:(length(θ) - Tar.extraparams)], autodiff) + nnsol = ode_dfdx(ltd, t, θ[1:(length(θ) - ltd.extraparams)], autodiff) vals = nnsol .- physsol + T = eltype(vals) - # N dimensional vector if N outputs for NN(each row has logpdf of u[i] where u is vector of dependant variables) + # N dimensional vector if N outputs for NN(each row has logpdf of u[i] where u is vector + # of dependant variables) return [logpdf( MvNormal(vals[i, :], - LinearAlgebra.Diagonal(abs2.(Tar.phystd[i] .* - ones(length(vals[i, :]))))), - zeros(length(vals[i, :]))) for i in 1:length(Tar.prob.u0)] + Diagonal(abs2.(T(ltd.phystd[i]) .* ones(T, length(vals[i, :]))))), + zeros(T, length(vals[i, :])) + ) for i in 1:length(ltd.prob.u0)] end """ Prior logpdf for NN parameters + ODE constants. """ -function priorweights(Tar::LogTargetDensity, θ) - allparams = Tar.priors - # nn weights - nnwparams = allparams[1] - - if Tar.extraparams > 0 - # Vector of ode parameters priors - invpriors = allparams[2:end] - - invlogpdf = sum( - logpdf(invpriors[length(θ) - i + 1], θ[i]) - for i in (length(θ) - Tar.extraparams + 1):length(θ); - init = 0.0) - - return (invlogpdf - + - logpdf(nnwparams, θ[1:(length(θ) - Tar.extraparams)])) - else - return logpdf(nnwparams, θ) - end -end +@views function priorweights(ltd::LogTargetDensity, θ) + allparams = ltd.priors + nnwparams = allparams[1] # nn weights -function generate_Tar(chain::Lux.AbstractExplicitLayer, init_params) - θ, st = Lux.setup(Random.default_rng(), chain) - return init_params, chain, st -end + ltd.extraparams ≤ 0 && return logpdf(nnwparams, θ) -function generate_Tar(chain::Lux.AbstractExplicitLayer, init_params::Nothing) - θ, st = Lux.setup(Random.default_rng(), chain) - return θ, chain, st -end + # Vector of ode parameters priors + invpriors = allparams[2:end] -""" -NN OUTPUT AT t,θ ~ phi(t,θ). -""" -function (f::LogTargetDensity{C, S})(t::AbstractVector, - θ) where {C <: Lux.AbstractExplicitLayer, S} - θ = vector_to_parameters(θ, f.init_params) - y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), t'), θ, f.st) - ChainRulesCore.@ignore_derivatives f.st = st - f.prob.u0 .+ (t' .- f.prob.tspan[1]) .* y + invlogpdf = sum( + logpdf(invpriors[length(θ) - i + 1], θ[i]) + for i in (length(θ) - ltd.extraparams + 1):length(θ)) + + return invlogpdf + logpdf(nnwparams, θ[1:(length(θ) - ltd.extraparams)]) end -function (f::LogTargetDensity{C, S})(t::Number, - θ) where {C <: Lux.AbstractExplicitLayer, S} - θ = vector_to_parameters(θ, f.init_params) - y, st = f.chain(adapt(parameterless_type(ComponentArrays.getdata(θ)), [t]), θ, f.st) - ChainRulesCore.@ignore_derivatives f.st = st - f.prob.u0 .+ (t .- f.prob.tspan[1]) .* y +function generate_ltd(chain::AbstractLuxLayer, init_params) + return init_params, chain, LuxCore.initialstates(Random.default_rng(), chain) end -""" -Similar to ode_dfdx() in NNODE. -""" -function NNodederi(phi::LogTargetDensity, t::AbstractVector, θ, autodiff::Bool) - if autodiff - hcat(ForwardDiff.derivative.(ti -> phi(ti, θ), t)...) - else - (phi(t .+ sqrt(eps(eltype(t))), θ) - phi(t, θ)) ./ sqrt(eps(eltype(t))) - end +function generate_ltd(chain::AbstractLuxLayer, ::Nothing) + θ, st = LuxCore.setup(Random.default_rng(), chain) + return θ, chain, st end function kernelchoice(Kernel, MCMCkwargs) if Kernel == HMCDA - δ, λ = MCMCkwargs[:δ], MCMCkwargs[:λ] - Kernel(δ, λ) + Kernel(MCMCkwargs[:δ], MCMCkwargs[:λ]) elseif Kernel == NUTS δ, max_depth, Δ_max = MCMCkwargs[:δ], MCMCkwargs[:max_depth], MCMCkwargs[:Δ_max] - Kernel(δ, max_depth = max_depth, Δ_max = Δ_max) - else - # HMC - n_leapfrog = MCMCkwargs[:n_leapfrog] - Kernel(n_leapfrog) + Kernel(δ; max_depth, Δ_max) + else # HMC + Kernel(MCMCkwargs[:n_leapfrog]) end end """ - ahmc_bayesian_pinn_ode(prob, chain; strategy = GridTraining, - dataset = [nothing],init_params = nothing, - draw_samples = 1000, physdt = 1 / 20.0f0,l2std = [0.05], - phystd = [0.05], phynewstd = [0.05], priorsNNw = (0.0, 2.0), - param = [], nchains = 1, autodiff = false, Kernel = HMC, - Adaptorkwargs = (Adaptor = StanHMCAdaptor, - Metric = DiagEuclideanMetric, - targetacceptancerate = 0.8), - Integratorkwargs = (Integrator = Leapfrog,), - MCMCkwargs = (n_leapfrog = 30,), - progress = false, verbose = false) + ahmc_bayesian_pinn_ode(prob, chain; strategy = GridTraining, dataset = [nothing], + init_params = nothing, draw_samples = 1000, physdt = 1 / 20.0f0, + l2std = [0.05], phystd = [0.05], phynewstd = [0.05], priorsNNw = (0.0, 2.0), + param = [], nchains = 1, autodiff = false, Kernel = HMC, + Adaptorkwargs = (Adaptor = StanHMCAdaptor, + Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), + Integratorkwargs = (Integrator = Leapfrog,), + MCMCkwargs = (n_leapfrog = 30,), progress = false, + verbose = false) !!! warn - Note that `ahmc_bayesian_pinn_ode()` only supports ODEs which are written in the out-of-place form, i.e. - `du = f(u,p,t)`, and not `f(du,u,p,t)`. If not declared out-of-place, then the `ahmc_bayesian_pinn_ode()` - will exit with an error. + Note that `ahmc_bayesian_pinn_ode()` only supports ODEs which are written in the + out-of-place form, i.e. `du = f(u,p,t)`, and not `f(du,u,p,t)`. If not declared + out-of-place, then `ahmc_bayesian_pinn_ode()` will exit with an error. ## Example @@ -463,22 +328,29 @@ Incase you are only solving the Equations for solution, do not provide dataset ## Keyword Arguments -* `strategy`: The training strategy used to choose the points for the evaluations. By default GridTraining is used with given physdt discretization. -* `init_params`: initial parameter values for BPINN (ideally for multiple chains different initializations preferred) +* `strategy`: The training strategy used to choose the points for the evaluations. By + default GridTraining is used with given physdt discretization. +* `init_params`: initial parameter values for BPINN (ideally for multiple chains different + initializations preferred) * `nchains`: number of chains you want to sample -* `draw_samples`: number of samples to be drawn in the MCMC algorithms (warmup samples are ~2/3 of draw samples) +* `draw_samples`: number of samples to be drawn in the MCMC algorithms (warmup samples are + ~2/3 of draw samples) * `l2std`: standard deviation of BPINN prediction against L2 losses/Dataset * `phystd`: standard deviation of BPINN prediction against Chosen Underlying ODE System * `phynewstd`: standard deviation of new loss func term -* `priorsNNw`: Tuple of (mean, std) for BPINN Network parameters. Weights and Biases of BPINN are Normal Distributions by default. +* `priorsNNw`: Tuple of (mean, std) for BPINN Network parameters. Weights and Biases of + BPINN are Normal Distributions by default. * `param`: Vector of chosen ODE parameters Distributions in case of Inverse problems. * `autodiff`: Boolean Value for choice of Derivative Backend(default is numerical) * `physdt`: Timestep for approximating ODE in it's Time domain. (1/20.0 by default) * `Kernel`: Choice of MCMC Sampling Algorithm (AdvancedHMC.jl implementations HMC/NUTS/HMCDA) -* `Integratorkwargs`: `Integrator`, `jitter_rate`, `tempering_rate`. Refer: https://turinglang.org/AdvancedHMC.jl/stable/ -* `Adaptorkwargs`: `Adaptor`, `Metric`, `targetacceptancerate`. Refer: https://turinglang.org/AdvancedHMC.jl/stable/ - Note: Target percentage(in decimal) of iterations in which the proposals are accepted (0.8 by default) -* `MCMCargs`: A NamedTuple containing all the chosen MCMC kernel's(HMC/NUTS/HMCDA) Arguments, as follows : +* `Integratorkwargs`: `Integrator`, `jitter_rate`, `tempering_rate`. + Refer: https://turinglang.org/AdvancedHMC.jl/stable/ +* `Adaptorkwargs`: `Adaptor`, `Metric`, `targetacceptancerate`. + Refer: https://turinglang.org/AdvancedHMC.jl/stable/ Note: Target percentage (in decimal) + of iterations in which the proposals are accepted (0.8 by default) +* `MCMCargs`: A NamedTuple containing all the chosen MCMC kernel's (HMC/NUTS/HMCDA) + Arguments, as follows : * `n_leapfrog`: number of leapfrog steps for HMC * `δ`: target acceptance probability for NUTS and HMCDA * `λ`: target trajectory length for HMCDA @@ -488,67 +360,53 @@ Incase you are only solving the Equations for solution, do not provide dataset * `progress`: controls whether to show the progress meter or not. * `verbose`: controls the verbosity. (Sample call args in AHMC) -## Warnings +!!! warning -* AdvancedHMC.jl is still developing convenience structs so might need changes on new releases. + AdvancedHMC.jl is still developing convenience structs so might need changes on new + releases. """ -function ahmc_bayesian_pinn_ode(prob::SciMLBase.ODEProblem, chain; - strategy = GridTraining, dataset = [nothing], - init_params = nothing, draw_samples = 1000, - physdt = 1 / 20.0, l2std = [0.05], - phystd = [0.05], phynewstd = [0.05], priorsNNw = (0.0, 2.0), - param = [], nchains = 1, autodiff = false, - Kernel = HMC, +function ahmc_bayesian_pinn_ode( + prob::SciMLBase.ODEProblem, chain; strategy = GridTraining, dataset = [nothing], + init_params = nothing, draw_samples = 1000, physdt = 1 / 20.0, l2std = [0.05], + phystd = [0.05], phynewstd = [0.05], priorsNNw = (0.0, 2.0), param = [], nchains = 1, + autodiff = false, Kernel = HMC, Adaptorkwargs = (Adaptor = StanHMCAdaptor, Metric = DiagEuclideanMetric, targetacceptancerate = 0.8), - Integratorkwargs = (Integrator = Leapfrog,), - MCMCkwargs = (n_leapfrog = 30,), - progress = false, verbose = false, - estim_collocate = false) - !(chain isa Lux.AbstractExplicitLayer) && - (chain = adapt(FromFluxAdaptor(false, false), chain)) - # NN parameter prior mean and variance(PriorsNN must be a tuple) - if isinplace(prob) - throw(error("The BPINN ODE solver only supports out-of-place ODE definitions, i.e. du=f(u,p,t).")) - end + Integratorkwargs = (Integrator = Leapfrog,), MCMCkwargs = (n_leapfrog = 30,), + progress = false, verbose = false, estim_collocate = false) + @assert !isinplace(prob) "The BPINN ODE solver only supports out-of-place ODE definitions, i.e. du=f(u,p,t)." + + chain isa AbstractLuxLayer || (chain = FromFluxAdaptor()(chain)) strategy = strategy == GridTraining ? strategy(physdt) : strategy if dataset != [nothing] && (length(dataset) < 2 || !(dataset isa Vector{<:Vector{<:AbstractFloat}})) - throw(error("Invalid dataset. dataset would be timeseries (x̂,t) where type: Vector{Vector{AbstractFloat}")) + error("Invalid dataset. dataset would be timeseries (x̂,t) where type: Vector{Vector{AbstractFloat}") end if dataset != [nothing] && param == [] println("Dataset is only needed for Parameter Estimation + Forward Problem, not in only Forward Problem case.") elseif dataset == [nothing] && param != [] - throw(error("Dataset Required for Parameter Estimation.")) + error("Dataset Required for Parameter Estimation.") end - if chain isa Lux.AbstractExplicitLayer - # Lux-Named Tuple - initial_nnθ, recon, st = generate_Tar(chain, init_params) - else - error("Only Lux.AbstractExplicitLayer Neural networks are supported") - end + initial_nnθ, chain, st = generate_ltd(chain, init_params) - if nchains > Threads.nthreads() - throw(error("number of chains is greater than available threads")) - elseif nchains < 1 - throw(error("number of chains must be greater than 1")) - end + @assert nchains≤Threads.nthreads() "number of chains is greater than available threads" + @assert nchains≥1 "number of chains must be greater than 1" # eltype(physdt) cause needs Float64 for find_good_stepsize # Lux chain(using component array later as vector_to_parameter need namedtuple) - initial_θ = collect(eltype(physdt), - vcat(ComponentArrays.ComponentArray(initial_nnθ))) + T = eltype(physdt) + initial_θ = getdata(ComponentArray{T}(initial_nnθ)) # adding ode parameter estimation nparameters = length(initial_θ) ninv = length(param) priors = [ - MvNormal(priorsNNw[1] * ones(nparameters), - LinearAlgebra.Diagonal(abs2.(priorsNNw[2] .* ones(nparameters)))) + MvNormal(T(priorsNNw[1]) * ones(T, nparameters), + Diagonal(abs2.(T(priorsNNw[2]) .* ones(T, nparameters)))) ] # append Ode params to all paramvector @@ -560,30 +418,25 @@ function ahmc_bayesian_pinn_ode(prob::SciMLBase.ODEProblem, chain; end t0 = prob.tspan[1] + smodel = StatefulLuxLayer{true}(chain, nothing, st) # dimensions would be total no of params,initial_nnθ for Lux namedTuples - ℓπ = LogTargetDensity(nparameters, prob, recon, st, strategy, dataset, priors, + ℓπ = LogTargetDensity(nparameters, prob, smodel, strategy, dataset, priors, phystd, phynewstd, l2std, autodiff, physdt, ninv, initial_nnθ, estim_collocate) - try - ℓπ(t0, initial_θ[1:(nparameters - ninv)]) - catch err - if isa(err, DimensionMismatch) - throw(DimensionMismatch("Dimensions of the initial u0 and chain should match")) - else - throw(err) + if verbose + @printf("Current Physics Log-likelihood: %g\n", physloglikelihood(ℓπ, initial_θ)) + @printf("Current Prior Log-likelihood: %g\n", priorweights(ℓπ, initial_θ)) + @printf("Current MSE against dataset Log-likelihood: %g\n", + L2LossData(ℓπ, initial_θ)) + if estim_collocate + @printf("Current gradient loss against dataset Log-likelihood: %g\n", + L2loss2(ℓπ, initial_θ)) end end - @info("Current Physics Log-likelihood : ", physloglikelihood(ℓπ, initial_θ)) - @info("Current Prior Log-likelihood : ", priorweights(ℓπ, initial_θ)) - @info("Current MSE against dataset Log-likelihood : ", L2LossData(ℓπ, initial_θ)) - if estim_collocate - @info("Current gradient loss against dataset Log-likelihood : ", - L2loss2(ℓπ, initial_θ)) - end - - Adaptor, Metric, targetacceptancerate = Adaptorkwargs[:Adaptor], - Adaptorkwargs[:Metric], Adaptorkwargs[:targetacceptancerate] + Adaptor = Adaptorkwargs[:Adaptor] + Metric = Adaptorkwargs[:Metric] + targetacceptancerate = Adaptorkwargs[:targetacceptancerate] # Define Hamiltonian system (nparameters ~ dimensionality of the sampling space) metric = Metric(nparameters) @@ -598,8 +451,10 @@ function ahmc_bayesian_pinn_ode(prob::SciMLBase.ODEProblem, chain; Threads.@threads for i in 1:nchains # each chain has different initial NNparameter values(better posterior exploration) - initial_θ = vcat(randn(nparameters - ninv), - initial_θ[(nparameters - ninv + 1):end]) + initial_θ = vcat( + randn(eltype(initial_θ), nparameters - ninv), + initial_θ[(nparameters - ninv + 1):end] + ) initial_ϵ = find_good_stepsize(hamiltonian, initial_θ) integrator = integratorchoice(Integratorkwargs, initial_ϵ) adaptor = adaptorchoice(Adaptor, MassMatrixAdaptor(metric), @@ -612,7 +467,7 @@ function ahmc_bayesian_pinn_ode(prob::SciMLBase.ODEProblem, chain; samplesc[i] = samples statsc[i] = stats - mcmc_chain = Chains(hcat(samples...)') + mcmc_chain = Chains(reduce(hcat, samples)') chains[i] = mcmc_chain end @@ -628,13 +483,17 @@ function ahmc_bayesian_pinn_ode(prob::SciMLBase.ODEProblem, chain; samples, stats = sample(hamiltonian, Kernel, initial_θ, draw_samples, adaptor; progress = progress, verbose = verbose) - @info("Sampling Complete.") - @info("Final Physics Log-likelihood : ", physloglikelihood(ℓπ, samples[end])) - @info("Final Prior Log-likelihood : ", priorweights(ℓπ, samples[end])) - @info("Final MSE against dataset Log-likelihood : ", L2LossData(ℓπ, samples[end])) - if estim_collocate - @info("Final gradient loss against dataset Log-likelihood : ", - L2loss2(ℓπ, samples[end])) + if verbose + println("Sampling Complete.") + @printf("Final Physics Log-likelihood: %g\n", + physloglikelihood(ℓπ, samples[end])) + @printf("Final Prior Log-likelihood: %g\n", priorweights(ℓπ, samples[end])) + @printf("Final MSE against dataset Log-likelihood: %g\n", + L2LossData(ℓπ, samples[end])) + if estim_collocate + @printf("Final gradient loss against dataset Log-likelihood: %g\n", + L2loss2(ℓπ, samples[end])) + end end # return a chain(basic chain),samples and stats diff --git a/src/discretize.jl b/src/discretize.jl index 7eb6c97af0..5187a0638a 100644 --- a/src/discretize.jl +++ b/src/discretize.jl @@ -23,23 +23,14 @@ to end end) -for Lux.AbstractExplicitLayer. +for Lux.AbstractLuxLayer. """ function build_symbolic_loss_function(pinnrep::PINNRepresentation, eqs; - eq_params = SciMLBase.NullParameters(), - param_estim = false, - default_p = nothing, - bc_indvars = pinnrep.indvars, - integrand = nothing, - dict_transformation_vars = nothing, - transformation_vars = nothing, + eq_params = SciMLBase.NullParameters(), param_estim = false, default_p = nothing, + bc_indvars = pinnrep.indvars, integrand = nothing, + dict_transformation_vars = nothing, transformation_vars = nothing, integrating_depvars = pinnrep.depvars) - @unpack indvars, depvars, dict_indvars, dict_depvars, dict_depvar_input, - phi, derivative, integral, - multioutput, init_params, strategy, eq_params, - param_estim, default_p = pinnrep - - eltypeθ = eltype(pinnrep.flat_init_params) + (; depvars, dict_depvars, dict_depvar_input, phi, derivative, integral, multioutput, init_params, strategy, eq_params, param_estim, default_p) = pinnrep if integrand isa Nothing loss_function = parse_equation(pinnrep, eqs) @@ -68,9 +59,6 @@ function build_symbolic_loss_function(pinnrep::PINNRepresentation, eqs; expr_θ = Expr[] expr_phi = Expr[] - acum = [0; accumulate(+, map(length, init_params))] - sep = [(acum[i] + 1):acum[i + 1] for i in 1:(length(acum) - 1)] - for i in eachindex(depvars) push!(expr_θ, :($θ.depvar.$(depvars[i]))) push!(expr_phi, :(phi[$i])) @@ -138,34 +126,28 @@ function build_symbolic_loss_function(pinnrep::PINNRepresentation, eqs; end let_ex = Expr(:let, vars_eq, vcat_expr_loss_functions) push!(ex.args, let_ex) - expr_loss_function = :(($vars) -> begin + return :(($vars) -> begin $ex end) end """ - build_loss_function(eqs, indvars, depvars, phi, derivative, init_params; bc_indvars=nothing) + build_loss_function(eqs, indvars, depvars, phi, derivative, init_params; + bc_indvars=nothing) Returns the body of loss function, which is the executable Julia function, for the main equation or boundary condition. """ function build_loss_function(pinnrep::PINNRepresentation, eqs, bc_indvars) - @unpack eq_params, param_estim, default_p, phi, derivative, integral = pinnrep + (; eq_params, param_estim, default_p, phi, derivative, integral) = pinnrep bc_indvars = bc_indvars === nothing ? pinnrep.indvars : bc_indvars - expr_loss_function = build_symbolic_loss_function(pinnrep, eqs; - bc_indvars = bc_indvars, - eq_params = eq_params, - param_estim = param_estim, - default_p = default_p) + expr_loss_function = build_symbolic_loss_function(pinnrep, eqs; bc_indvars, eq_params, + param_estim, default_p) u = get_u() _loss_function = @RuntimeGeneratedFunction(expr_loss_function) - loss_function = (cord, θ) -> begin - _loss_function(cord, θ, phi, derivative, integral, u, - default_p) - end - return loss_function + return (cord, θ) -> _loss_function(cord, θ, phi, derivative, integral, u, default_p) end """ @@ -178,8 +160,7 @@ function generate_training_sets end function generate_training_sets(domains, dx, eqs, bcs, eltypeθ, _indvars::Array, _depvars::Array) - depvars, indvars, dict_indvars, dict_depvars, dict_depvar_input = get_vars(_indvars, - _depvars) + _, _, dict_indvars, dict_depvars, _ = get_vars(_indvars, _depvars) return generate_training_sets(domains, dx, eqs, bcs, eltypeθ, dict_indvars, dict_depvars) end @@ -187,11 +168,7 @@ end # Generate training set in the domain and on the boundary function generate_training_sets(domains, dx, eqs, bcs, eltypeθ, dict_indvars::Dict, dict_depvars::Dict) - if dx isa Array - dxs = dx - else - dxs = fill(dx, length(domains)) - end + dxs = dx isa Array ? dx : fill(dx, length(domains)) spans = [infimum(d.domain):dx:supremum(d.domain) for (d, dx) in zip(domains, dxs)] dict_var_span = Dict([Symbol(d.variables) => infimum(d.domain):dx:supremum(d.domain) @@ -201,12 +178,8 @@ function generate_training_sets(domains, dx, eqs, bcs, eltypeθ, dict_indvars::D bound_vars = get_variables(bcs, dict_indvars, dict_depvars) dif = [eltypeθ[] for i in 1:size(domains)[1]] - for _args in bound_vars - for (i, x) in enumerate(_args) - if x isa Number - push!(dif[i], x) - end - end + for _args in bound_vars, (i, x) in enumerate(_args) + x isa Number && push!(dif[i], x) end cord_train_set = collect.(spans) bc_data = map(zip(dif, cord_train_set)) do (d, c) @@ -216,24 +189,20 @@ function generate_training_sets(domains, dx, eqs, bcs, eltypeθ, dict_indvars::D dict_var_span_ = Dict([Symbol(d.variables) => bc for (d, bc) in zip(domains, bc_data)]) bcs_train_sets = map(bound_args) do bt - span = map(b -> get(dict_var_span, b, b), bt) - _set = adapt(eltypeθ, - hcat(vec(map(points -> collect(points), Iterators.product(span...)))...)) + span = get.((dict_var_span,), bt, bt) + return reduce(hcat, vec(map(collect, Iterators.product(span...)))) |> + EltypeAdaptor{eltypeθ}() end - pde_vars = get_variables(eqs, dict_indvars, dict_depvars) pde_args = get_argument(eqs, dict_indvars, dict_depvars) - pde_train_set = adapt(eltypeθ, - hcat(vec(map(points -> collect(points), - Iterators.product(bc_data...)))...)) - pde_train_sets = map(pde_args) do bt - span = map(b -> get(dict_var_span_, b, b), bt) - _set = adapt(eltypeθ, - hcat(vec(map(points -> collect(points), Iterators.product(span...)))...)) + span = get.((dict_var_span_,), bt, bt) + return reduce(hcat, vec(map(collect, Iterators.product(span...)))) |> + EltypeAdaptor{eltypeθ}() end - [pde_train_sets, bcs_train_sets] + + return [pde_train_sets, bcs_train_sets] end """ @@ -245,32 +214,33 @@ training strategy: StochasticTraining, QuasiRandomTraining, QuadratureTraining. function get_bounds end function get_bounds(domains, eqs, bcs, eltypeθ, _indvars::Array, _depvars::Array, strategy) - depvars, indvars, dict_indvars, dict_depvars, dict_depvar_input = get_vars(_indvars, - _depvars) + _, _, dict_indvars, dict_depvars, _ = get_vars(_indvars, _depvars) return get_bounds(domains, eqs, bcs, eltypeθ, dict_indvars, dict_depvars, strategy) end function get_bounds(domains, eqs, bcs, eltypeθ, _indvars::Array, _depvars::Array, strategy::QuadratureTraining) - depvars, indvars, dict_indvars, dict_depvars, dict_depvar_input = get_vars(_indvars, - _depvars) + _, _, dict_indvars, dict_depvars, _ = get_vars(_indvars, _depvars) return get_bounds(domains, eqs, bcs, eltypeθ, dict_indvars, dict_depvars, strategy) end function get_bounds(domains, eqs, bcs, eltypeθ, dict_indvars, dict_depvars, - strategy::QuadratureTraining) + ::QuadratureTraining) dict_lower_bound = Dict([Symbol(d.variables) => infimum(d.domain) for d in domains]) dict_upper_bound = Dict([Symbol(d.variables) => supremum(d.domain) for d in domains]) pde_args = get_argument(eqs, dict_indvars, dict_depvars) + ϵ = cbrt(eps(eltypeθ)) + eltype_adaptor = EltypeAdaptor{eltypeθ}() + pde_lower_bounds = map(pde_args) do pd - span = map(p -> get(dict_lower_bound, p, p), pd) - map(s -> adapt(eltypeθ, s) + cbrt(eps(eltypeθ)), span) + span = get.((dict_lower_bound,), pd, pd) |> eltype_adaptor + return span .+ ϵ end pde_upper_bounds = map(pde_args) do pd - span = map(p -> get(dict_upper_bound, p, p), pd) - map(s -> adapt(eltypeθ, s) - cbrt(eps(eltypeθ)), span) + span = get.((dict_upper_bound,), pd, pd) |> eltype_adaptor + return span .+ ϵ end pde_bounds = [pde_lower_bounds, pde_upper_bounds] @@ -284,42 +254,39 @@ function get_bounds(domains, eqs, bcs, eltypeθ, dict_indvars, dict_depvars, end bcs_bounds = [bcs_lower_bounds, bcs_upper_bounds] - [pde_bounds, bcs_bounds] + return [pde_bounds, bcs_bounds] end function get_bounds(domains, eqs, bcs, eltypeθ, dict_indvars, dict_depvars, strategy) dx = 1 / strategy.points dict_span = Dict([Symbol(d.variables) => [ - infimum(d.domain) + dx, - supremum(d.domain) - dx - ] for d in domains]) + infimum(d.domain) + dx, supremum(d.domain) - dx] for d in domains]) - # pde_bounds = [[infimum(d.domain),supremum(d.domain)] for d in domains] pde_args = get_argument(eqs, dict_indvars, dict_depvars) pde_bounds = map(pde_args) do pde_arg bds = mapreduce(s -> get(dict_span, s, fill(s, 2)), hcat, pde_arg) bds = eltypeθ.(bds) - bds[1, :], bds[2, :] + return bds[1, :], bds[2, :] end bound_args = get_argument(bcs, dict_indvars, dict_depvars) bcs_bounds = map(bound_args) do bound_arg bds = mapreduce(s -> get(dict_span, s, fill(s, 2)), hcat, bound_arg) bds = eltypeθ.(bds) - bds[1, :], bds[2, :] + return bds[1, :], bds[2, :] end + return pde_bounds, bcs_bounds end function get_numeric_integral(pinnrep::PINNRepresentation) - @unpack strategy, indvars, depvars, multioutput, derivative, - depvars, indvars, dict_indvars, dict_depvars = pinnrep + (; strategy, indvars, depvars, derivative, depvars, indvars, dict_indvars, dict_depvars) = pinnrep - integral = (u, cord, phi, integrating_var_id, integrand_func, lb, ub, θ; strategy = strategy, indvars = indvars, depvars = depvars, dict_indvars = dict_indvars, dict_depvars = dict_depvars) -> begin + return (u, cord, phi, integrating_var_id, integrand_func, lb, ub, θ; strategy = strategy, indvars = indvars, depvars = depvars, dict_indvars = dict_indvars, dict_depvars = dict_depvars) -> begin function integration_(cord, lb, ub, θ) cord_ = cord function integrand_(x, p) - ChainRulesCore.@ignore_derivatives @views(cord_[integrating_var_id]) .= x + @ignore_derivatives cord_[integrating_var_id] .= x return integrand_func(cord_, p, phi, derivative, nothing, u, nothing) end prob_ = IntegralProblem(integrand_, (lb, ub), θ) @@ -332,24 +299,22 @@ function get_numeric_integral(pinnrep::PINNRepresentation) ub_ = zeros(size(ub)[1], size(cord)[2]) for (i, l) in enumerate(lb) if l isa Number - ChainRulesCore.@ignore_derivatives lb_[i, :] = fill(l, 1, size(cord)[2]) + @ignore_derivatives lb_[i, :] .= l else - ChainRulesCore.@ignore_derivatives lb_[i, :] = l(cord, θ, phi, derivative, - nothing, u, nothing) + @ignore_derivatives lb_[i, :] = l( + cord, θ, phi, derivative, nothing, u, nothing) end end for (i, u_) in enumerate(ub) if u_ isa Number - ChainRulesCore.@ignore_derivatives ub_[i, :] = fill(u_, 1, size(cord)[2]) + @ignore_derivatives ub_[i, :] .= u_ else - ChainRulesCore.@ignore_derivatives ub_[i, :] = u_(cord, θ, phi, derivative, + @ignore_derivatives ub_[i, :] = u_(cord, θ, phi, derivative, nothing, u, nothing) end end integration_arr = Matrix{Float64}(undef, 1, 0) - for i in 1:size(cord)[2] - # ub__ = @Zygote.ignore getindex(ub_, :, i) - # lb__ = @Zygote.ignore getindex(lb_, :, i) + for i in 1:size(cord, 2) integration_arr = hcat(integration_arr, integration_(cord[:, i], lb_[:, i], ub_[:, i], θ)) end @@ -364,33 +329,25 @@ end It transforms a symbolic description of a ModelingToolkit-defined `PDESystem` into a `PINNRepresentation` which holds the pieces required to build an `OptimizationProblem` for [Optimization.jl](https://docs.sciml.ai/Optimization/stable) or a Likelihood Function -used for HMC based Posterior Sampling Algorithms [AdvancedHMC.jl](https://turinglang.org/AdvancedHMC.jl/stable/) -which is later optimized upon to give Solution or the Solution Distribution of the PDE. +used for HMC based Posterior Sampling Algorithms +[AdvancedHMC.jl](https://turinglang.org/AdvancedHMC.jl/stable/) which is later optimized +upon to give Solution or the Solution Distribution of the PDE. For more information, see `discretize` and `PINNRepresentation`. """ -function SciMLBase.symbolic_discretize(pde_system::PDESystem, - discretization::AbstractPINN) - eqs = pde_system.eqs - bcs = pde_system.bcs - chain = discretization.chain - - domains = pde_system.domain +function SciMLBase.symbolic_discretize(pde_system::PDESystem, discretization::AbstractPINN) + (; eqs, bcs, domain) = pde_system eq_params = pde_system.ps defaults = pde_system.defaults - default_p = eq_params == SciMLBase.NullParameters() ? nothing : - [defaults[ep] for ep in eq_params] - - param_estim = discretization.param_estim - additional_loss = discretization.additional_loss + (; chain, param_estim, additional_loss, multioutput, init_params, phi, derivative, strategy, logger, iteration, self_increment) = discretization + (; log_frequency) = discretization.log_options adaloss = discretization.adaptive_loss - depvars, indvars, dict_indvars, dict_depvars, dict_depvar_input = get_vars( - pde_system.indvars, - pde_system.depvars) + default_p = eq_params isa SciMLBase.NullParameters ? nothing : + [defaults[ep] for ep in eq_params] - multioutput = discretization.multioutput - init_params = discretization.init_params + depvars, indvars, dict_indvars, dict_depvars, dict_depvar_input = get_vars( + pde_system.indvars, pde_system.depvars) if init_params === nothing # Use the initialization of the neural network framework @@ -398,70 +355,41 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, # This is done because Float64 is almost always better for these applications if chain isa AbstractArray x = map(chain) do x - _x = ComponentArrays.ComponentArray(Lux.initialparameters( - Random.default_rng(), - x)) - Float64.(_x) # No ComponentArray GPU support + ComponentArray{Float64}(LuxCore.initialparameters(Random.default_rng(), x)) end names = ntuple(i -> depvars[i], length(chain)) - init_params = ComponentArrays.ComponentArray(NamedTuple{names}(i - for i in x)) + init_params = ComponentArray(NamedTuple{names}(Tuple(x))) else - init_params = Float64.(ComponentArrays.ComponentArray(Lux.initialparameters( - Random.default_rng(), - chain))) + init_params = ComponentArray{Float64}(LuxCore.initialparameters( + Random.default_rng(), chain)) end - else - init_params = init_params end - flat_init_params = if init_params isa ComponentArrays.ComponentArray + flat_init_params = if init_params isa ComponentArray init_params elseif multioutput @assert length(init_params) == length(depvars) names = ntuple(i -> depvars[i], length(init_params)) - x = ComponentArrays.ComponentArray(NamedTuple{names}(i for i in init_params)) + x = ComponentArray(NamedTuple{names}(Tuple(init_params))) else - ComponentArrays.ComponentArray(init_params) + ComponentArray(init_params) end - flat_init_params = if param_estim == false && multioutput - ComponentArrays.ComponentArray(; depvar = flat_init_params) - elseif param_estim == false && !multioutput - flat_init_params + flat_init_params = if !param_estim + multioutput ? ComponentArray(; depvar = flat_init_params) : flat_init_params else - ComponentArrays.ComponentArray(; depvar = flat_init_params, p = default_p) + ComponentArray(; depvar = flat_init_params, p = default_p) end - eltypeθ = eltype(flat_init_params) - - if adaloss === nothing - adaloss = NonAdaptiveLoss{eltypeθ}() + if length(flat_init_params) == 0 && !Base.isconcretetype(eltype(flat_init_params)) + flat_init_params = ComponentArray( + convert(AbstractArray{Float64}, getdata(flat_init_params)), + getaxes(flat_init_params)) end - phi = discretization.phi + adaloss === nothing && (adaloss = NonAdaptiveLoss{eltype(flat_init_params)}()) - if (phi isa Vector && phi[1].f isa Lux.AbstractExplicitLayer) - for ϕ in phi - ϕ.st = adapt(parameterless_type(ComponentArrays.getdata(flat_init_params)), - ϕ.st) - end - elseif (!(phi isa Vector) && phi.f isa Lux.AbstractExplicitLayer) - phi.st = adapt(parameterless_type(ComponentArrays.getdata(flat_init_params)), - phi.st) - end - - derivative = discretization.derivative - strategy = discretization.strategy - - logger = discretization.logger - log_frequency = discretization.log_options.log_frequency - iteration = discretization.iteration - self_increment = discretization.self_increment - - if !(eqs isa Array) - eqs = [eqs] - end + eqs isa Array || (eqs = [eqs]) pde_indvars = if strategy isa QuadratureTraining get_argument(eqs, dict_indvars, dict_depvars) @@ -478,7 +406,7 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, pde_integration_vars = get_integration_variables(eqs, dict_indvars, dict_depvars) bc_integration_vars = get_integration_variables(bcs, dict_indvars, dict_depvars) - pinnrep = PINNRepresentation(eqs, bcs, domains, eq_params, defaults, default_p, + pinnrep = PINNRepresentation(eqs, bcs, domain, eq_params, defaults, default_p, param_estim, additional_loss, adaloss, depvars, indvars, dict_indvars, dict_depvars, dict_depvar_input, logger, multioutput, iteration, init_params, flat_init_params, phi, @@ -503,39 +431,32 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, pinnrep.symbolic_bc_loss_functions = symbolic_bc_loss_functions datafree_pde_loss_functions = [build_loss_function(pinnrep, eq, pde_indvar) - for (eq, pde_indvar, integration_indvar) in zip(eqs, - pde_indvars, - pde_integration_vars)] + for (eq, pde_indvar) in zip(eqs, pde_indvars)] datafree_bc_loss_functions = [build_loss_function(pinnrep, bc, bc_indvar) - for (bc, bc_indvar, integration_indvar) in zip(bcs, - bc_indvars, - bc_integration_vars)] + for (bc, bc_indvar) in zip(bcs, bc_indvars)] + + pde_loss_functions, bc_loss_functions = merge_strategy_with_loss_function(pinnrep, + strategy, datafree_pde_loss_functions, datafree_bc_loss_functions) + + # setup for all adaptive losses + num_pde_losses = length(pde_loss_functions) + num_bc_losses = length(bc_loss_functions) + # assume one single additional loss function if there is one. this means that the user needs to lump all their functions into a single one, + num_additional_loss = convert(Int, additional_loss !== nothing) - function get_likelihood_estimate_function(discretization::PhysicsInformedNN) - pde_loss_functions, bc_loss_functions = merge_strategy_with_loss_function(pinnrep, - strategy, - datafree_pde_loss_functions, - datafree_bc_loss_functions) - # setup for all adaptive losses - num_pde_losses = length(pde_loss_functions) - num_bc_losses = length(bc_loss_functions) - # assume one single additional loss function if there is one. this means that the user needs to lump all their functions into a single one, - num_additional_loss = additional_loss isa Nothing ? 0 : 1 - - adaloss_T = eltype(adaloss.pde_loss_weights) - - # this will error if the user has provided a number of initial weights that is more than 1 and doesn't match the number of loss functions - adaloss.pde_loss_weights = ones(adaloss_T, num_pde_losses) .* - adaloss.pde_loss_weights - adaloss.bc_loss_weights = ones(adaloss_T, num_bc_losses) .* adaloss.bc_loss_weights - adaloss.additional_loss_weights = ones(adaloss_T, num_additional_loss) .* - adaloss.additional_loss_weights + adaloss_T = eltype(adaloss.pde_loss_weights) + + # this will error if the user has provided a number of initial weights that is more than 1 and doesn't match the number of loss functions + adaloss.pde_loss_weights = ones(adaloss_T, num_pde_losses) .* adaloss.pde_loss_weights + adaloss.bc_loss_weights = ones(adaloss_T, num_bc_losses) .* adaloss.bc_loss_weights + adaloss.additional_loss_weights = ones(adaloss_T, num_additional_loss) .* + adaloss.additional_loss_weights reweight_losses_func = generate_adaptive_loss_function(pinnrep, adaloss, - pde_loss_functions, - bc_loss_functions) + pde_loss_functions, bc_loss_functions) + function get_likelihood_estimate_function(::PhysicsInformedNN) function full_loss_function(θ, p) # the aggregation happens on cpu even if the losses are gpu, probably fine since it's only a few of them pde_losses = [pde_loss_function(θ) for pde_loss_function in pde_loss_functions] @@ -543,13 +464,12 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, # this is kind of a hack, and means that whenever the outer function is evaluated the increment goes up, even if it's not being optimized # that's why we prefer the user to maintain the increment in the outer loop callback during optimization - ChainRulesCore.@ignore_derivatives if self_increment - iteration[1] += 1 + @ignore_derivatives if self_increment + iteration[] += 1 end - ChainRulesCore.@ignore_derivatives begin - reweight_losses_func(θ, pde_losses, - bc_losses) + @ignore_derivatives begin + reweight_losses_func(θ, pde_losses, bc_losses) end weighted_pde_losses = adaloss.pde_loss_weights .* pde_losses @@ -563,126 +483,87 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, full_weighted_loss = if additional_loss isa Nothing weighted_loss_before_additional else - function _additional_loss(phi, θ) - (θ_, p_) = if (param_estim == true) - θ.depvar, θ.p - else - θ, nothing - end - return additional_loss(phi, θ_, p_) - end + (θ_, p_) = param_estim ? (θ.depvar, θ.p) : (θ, nothing) + _additional_loss = additional_loss(phi, θ_, p_) weighted_additional_loss_val = adaloss.additional_loss_weights[1] * - _additional_loss(phi, θ) + _additional_loss weighted_loss_before_additional + weighted_additional_loss_val end - ChainRulesCore.@ignore_derivatives begin - if iteration[1] % log_frequency == 0 + @ignore_derivatives begin + if iteration[] % log_frequency == 0 logvector(pinnrep.logger, pde_losses, "unweighted_loss/pde_losses", - iteration[1]) - logvector(pinnrep.logger, - bc_losses, - "unweighted_loss/bc_losses", - iteration[1]) + iteration[]) + logvector(pinnrep.logger, bc_losses, "unweighted_loss/bc_losses", + iteration[]) logvector(pinnrep.logger, weighted_pde_losses, - "weighted_loss/weighted_pde_losses", - iteration[1]) + "weighted_loss/weighted_pde_losses", iteration[]) logvector(pinnrep.logger, weighted_bc_losses, - "weighted_loss/weighted_bc_losses", - iteration[1]) - if !(additional_loss isa Nothing) + "weighted_loss/weighted_bc_losses", iteration[]) + if additional_loss !== nothing logscalar(pinnrep.logger, weighted_additional_loss_val, - "weighted_loss/weighted_additional_loss", iteration[1]) + "weighted_loss/weighted_additional_loss", iteration[]) end logscalar(pinnrep.logger, sum_weighted_pde_losses, - "weighted_loss/sum_weighted_pde_losses", iteration[1]) + "weighted_loss/sum_weighted_pde_losses", iteration[]) logscalar(pinnrep.logger, sum_weighted_bc_losses, - "weighted_loss/sum_weighted_bc_losses", iteration[1]) + "weighted_loss/sum_weighted_bc_losses", iteration[]) logscalar(pinnrep.logger, full_weighted_loss, - "weighted_loss/full_weighted_loss", - iteration[1]) + "weighted_loss/full_weighted_loss", iteration[]) logvector(pinnrep.logger, adaloss.pde_loss_weights, - "adaptive_loss/pde_loss_weights", - iteration[1]) + "adaptive_loss/pde_loss_weights", iteration[]) logvector(pinnrep.logger, adaloss.bc_loss_weights, - "adaptive_loss/bc_loss_weights", - iteration[1]) + "adaptive_loss/bc_loss_weights", iteration[]) end end return full_weighted_loss end - return bc_loss_functions, pde_loss_functions, full_loss_function + return full_loss_function end function get_likelihood_estimate_function(discretization::BayesianPINN) - # Because separate reweighting code section needed and loglikelihood is pointwise independent - pde_loss_functions, bc_loss_functions = merge_strategy_with_loglikelihood_function( - pinnrep, - strategy, - datafree_pde_loss_functions, - datafree_bc_loss_functions) - - # setup for all adaptive losses - num_pde_losses = length(pde_loss_functions) - num_bc_losses = length(bc_loss_functions) - # assume one single additional loss function if there is one. this means that the user needs to lump all their functions into a single one, - num_additional_loss = additional_loss isa Nothing ? 0 : 1 - - adaloss_T = eltype(adaloss.pde_loss_weights) - - # this will error if the user has provided a number of initial weights that is more than 1 and doesn't match the number of loss functions - adaloss.pde_loss_weights = ones(adaloss_T, num_pde_losses) .* - adaloss.pde_loss_weights - adaloss.bc_loss_weights = ones(adaloss_T, num_bc_losses) .* adaloss.bc_loss_weights - adaloss.additional_loss_weights = ones(adaloss_T, num_additional_loss) .* - adaloss.additional_loss_weights - - reweight_losses_func = generate_adaptive_loss_function(pinnrep, adaloss, - pde_loss_functions, - bc_loss_functions) - dataset_pde, dataset_bc = discretization.dataset - dataset_pde = dataset_pde isa Nothing ? dataset_pde : get_dataset_train_points(eqs, dataset_pde, pinnrep) - dataset_bc = dataset_bc isa Nothing ? dataset_bc : get_dataset_train_points(eqs, dataset_bc, pinnrep) # required as Physics loss also needed on the discrete dataset domain points # data points are discrete and so by default GridTraining loss applies - # passing placeholder dx with GridTraining, it uses dataset points irl - datapde_loss_functions, databc_loss_functions = merge_strategy_with_loglikelihood_function( - pinnrep, - GridTraining(0.1), - datafree_pde_loss_functions, - datafree_bc_loss_functions, - train_sets_pde = dataset_pde, - train_sets_bc = dataset_bc) + # passing placeholder dx with GridTraining, it uses data points irl + datapde_loss_functions, databc_loss_functions = if dataset_bc !== nothing || + dataset_pde !== nothing + merge_strategy_with_loglikelihood_function(pinnrep, GridTraining(0.1), + datafree_pde_loss_functions, datafree_bc_loss_functions, + train_sets_pde = dataset_pde, train_sets_bc = dataset_bc) + else + nothing, nothing + end function full_loss_function(θ, allstd::Vector{Vector{Float64}}) - stdpdes, stdbcs, stdextra, stdpdesnew = allstd + stdpdes, stdbcs, stdextra = allstd # the aggregation happens on cpu even if the losses are gpu, probably fine since it's only a few of them + pde_loglikelihoods = [logpdf(Normal(0, stdpdes[i]), pde_loss_function(θ)) + for (i, pde_loss_function) in enumerate(pde_loss_functions)] - pde_loglikelihoods = sum([pde_loss_function(θ, stdpdes[i]) - for (i, pde_loss_function) in enumerate(pde_loss_functions)]) - bc_loglikelihoods = sum([bc_loss_function(θ, stdbcs[j]) - for (j, bc_loss_function) in enumerate(bc_loss_functions)]) + bc_loglikelihoods = [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) + for (j, bc_loss_function) in enumerate(bc_loss_functions)] if !(datapde_loss_functions isa Nothing) - pde_loglikelihoods += sum([datapde_loss_function(θ, stdpdes[i]) - for (i, datapde_loss_function) in enumerate(datapde_loss_functions)]) + pde_loglikelihoods += [logpdf(Normal(0, stdpdes[j]), pde_loss_function(θ)) + for (j, pde_loss_function) in enumerate(datapde_loss_functions)] end + if !(databc_loss_functions isa Nothing) - bc_loglikelihoods += sum([databc_loss_function(θ, stdbcs[j]) - for (j, databc_loss_function) in enumerate(databc_loss_functions)]) + bc_loglikelihoods += [logpdf(Normal(0, stdbcs[j]), bc_loss_function(θ)) + for (j, bc_loss_function) in enumerate(databc_loss_functions)] end # this is kind of a hack, and means that whenever the outer function is evaluated the increment goes up, even if it's not being optimized # that's why we prefer the user to maintain the increment in the outer loop callback during optimization - ChainRulesCore.@ignore_derivatives if self_increment - iteration[1] += 1 + @ignore_derivatives if self_increment + iteration[] += 1 end - ChainRulesCore.@ignore_derivatives begin + @ignore_derivatives begin reweight_losses_func(θ, pde_loglikelihoods, bc_loglikelihoods) end @@ -698,17 +579,9 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, full_weighted_loglikelihood = if additional_loss isa Nothing weighted_loglikelihood_before_additional else - function _additional_loss(phi, θ) - (θ_, p_) = if (param_estim == true) - θ.depvar, θ.p - else - θ, nothing - end - return additional_loss(phi, θ_, p_) - end - - _additional_loglikelihood = logpdf(Normal(0, stdextra), - _additional_loss(phi, θ)) + (θ_, p_) = param_estim ? (θ.depvar, θ.p) : (θ, nothing) + _additional_loss = additional_loss(phi, θ_, p_) + _additional_loglikelihood = logpdf(Normal(0, stdextra), _additional_loss) weighted_additional_loglikelihood = adaloss.additional_loss_weights[1] * _additional_loglikelihood @@ -719,15 +592,13 @@ function SciMLBase.symbolic_discretize(pde_system::PDESystem, return full_weighted_loglikelihood end - return bc_loss_functions, pde_loss_functions, full_loss_function + return full_loss_function end - bc_loss_functions, pde_loss_functions, full_loss_function = get_likelihood_estimate_function(discretization) - + full_loss_function = get_likelihood_estimate_function(discretization) pinnrep.loss_functions = PINNLossFunctions(bc_loss_functions, pde_loss_functions, - full_loss_function, additional_loss, - datafree_pde_loss_functions, - datafree_bc_loss_functions) + full_loss_function, additional_loss, datafree_pde_loss_functions, + datafree_bc_loss_functions) return pinnrep end @@ -736,12 +607,11 @@ end prob = discretize(pde_system::PDESystem, discretization::PhysicsInformedNN) Transforms a symbolic description of a ModelingToolkit-defined `PDESystem` and generates -an `OptimizationProblem` for [Optimization.jl](https://docs.sciml.ai/Optimization/stable/) whose -solution is the solution to the PDE. +an `OptimizationProblem` for [Optimization.jl](https://docs.sciml.ai/Optimization/stable/) +whose solution is the solution to the PDE. """ function SciMLBase.discretize(pde_system::PDESystem, discretization::PhysicsInformedNN) pinnrep = symbolic_discretize(pde_system, discretization) - f = OptimizationFunction(pinnrep.loss_functions.full_loss_function, - Optimization.AutoZygote()) - Optimization.OptimizationProblem(f, pinnrep.flat_init_params) -end + f = OptimizationFunction(pinnrep.loss_functions.full_loss_function, AutoZygote()) + return Optimization.OptimizationProblem(f, pinnrep.flat_init_params) +end \ No newline at end of file