updates from ks, burg runs

examples/autodecoder.jl

@@ -191,7 +191,7 @@ function train_SNF(
     NN = AutoDecoder(decoder, metadata._Ns, l; init_weight = init_code)
 
     _batchsize = isnothing(_batchsize) ? numobs(_data) ÷ 100 : _batchsize
-    batchsize_ = isnothing(batchsize_) ? numobs(_data)       : batchsize_
+    batchsize_ = isnothing(batchsize_) ? numobs(_data) ÷ 1   : batchsize_
 
     lossfun = regularize_autodecoder(mse; σ2inv, α, λ1, λ2)
 
@@ -201,18 +201,19 @@ function train_SNF(
     lrs = (1f-3, 5f-4, 2f-4, 1f-4, 5f-5, 2f-5, 1f-5,)
     Nlrs = length(lrs)
 
-    weightdecay = if isnothing(WeightDecayOpt) | (WeightDecayOpt === DecoderWeightDecay)
-        ca_axes = p_axes(NN; rng)
-        DecoderWeightDecay(0f0, ca_axes)
-    elseif WeightDecayOpt === IdxWeightDecay
-        decoder_idx = if isnothing(weight_decay_ifunc) | (weight_decay_ifunc === decoder_indices)
-            # decoder_W_indices(NN; rng) # WD on decoder weights
-            decoder_indices(NN; rng)   # WD on decoder weights and biases
+    weightdecay = if isnothing(WeightDecayOpt) | (WeightDecayOpt === IdxWeightDecay)
+
+        decoder_idx = if isnothing(weight_decay_ifunc) | (weight_decay_ifunc === decoder_W_indices)
+            decoder_W_indices(NN; rng)   # WD on decoder weights and biases
         else
             weight_decay_ifunc(NN; rng)
         end
 
         IdxWeightDecay(0f0, decoder_idx)
+
+    elseif WeightDecayOpt === DecoderWeightDecay
+        ca_axes = p_axes(NN; rng)
+        DecoderWeightDecay(0f0, ca_axes)
     else
         error("Unsupported weight decay algorithm")
     end
@@ -864,7 +865,7 @@ function eval_model(
     model::NeuralEmbeddingModel,
     x::Tuple,
     p::AbstractArray;
-    batchsize = numobs(x) ÷ 101,
+    batchsize = numobs(x) ÷ 100,
     device = Lux.cpu_device(),
 )
     loader = MLUtils.DataLoader(x; batchsize, shuffle = false, partial = true)

examples/burgers_fourier1D/autodecode.jl

@@ -1,52 +1,52 @@
 #
-"""
-Train an autoencoder on 1D Burgers data
-"""
-
 using GeometryLearning
 include(joinpath(pkgdir(GeometryLearning), "examples", "autodecoder.jl"))
 #======================================================#
 
 rng = Random.default_rng()
 Random.seed!(rng, 460)
 
+prob = BurgersViscous1D(1f-4)
 device = Lux.gpu_device()
 datafile = joinpath(@__DIR__, "burg_visc_re10k", "data.jld2")
-
 modeldir = joinpath(@__DIR__, "dump")
 modelfile = joinpath(modeldir, "model_08.jld2")
 
-prob = BurgersViscous1D(1f-4)
-
 E = 3500  # epochs
 l = 8     # latent
 h = 5     # num hidden
 w = 128   # width
 
-λ1, λ2 = 0f0, 0f0     # L1 / L2 reg
-σ2inv, α = 1f-2, 0f-0 # code / Lipschitz regularization
-weight_decays = 5f-2  # AdamW weight decay
+λ1, λ2 = 0f0, 0f0
+σ2inv, α = 1f-1, 0f-0 # 1f-1, 1f-3
+weight_decays = 1f-2  # 1f-2
+WeightDecayOpt = IdxWeightDecay
+weight_decay_ifunc = decoder_W_indices
 
-_Ib, Ib_ = [2], [2]
-Ix = LinRange(1, 8192, 1024) .|> Base.Fix1(round, Int)
-makedata_kws = (; Ix, _Ib, Ib_, _It = :, It_ = :)
+_Ib, Ib_ = [5,], [5,] # 1, 2 work
+Ix  = LinRange(1, 8192, 1024) .|> Base.Fix1(round, Int)
+_It = Colon() # LinRange(1, 1000, 200 ) .|> Base.Fix1(round, Int)
+makedata_kws = (; Ix, _Ib, Ib_, _It = _It, It_ = :)
 
 ## train
-# isdir(modeldir) && rm(modeldir, recursive = true)
-# train_SNF(datafile, modeldir, l, h, w, E;
-#     rng, warmup = true,
-#     λ1, λ2, σ2inv, α, weight_decays, device, makedata_kws,
-# )
+isdir(modeldir) && rm(modeldir, recursive = true)
+train_SNF(datafile, modeldir, l, h, w, E;
+    rng, warmup = true, makedata_kws,
+    λ1, λ2, σ2inv, α, weight_decays, device,
+    WeightDecayOpt, weight_decay_ifunc,
+)
 
-## process
 outdir = joinpath(modeldir, "results")
 postprocess_SNF(prob, datafile, modelfile, outdir; rng, device,
     makeplot = true, verbose = true)
-x, t, ud, up, _ = evolve_SNF(prob, datafile, modelfile, _Ib[1]; rng, device, verbose = true)
+x, t, ud, up, _ = evolve_SNF(prob, datafile, modelfile, Ib_[1]; rng, device, verbose = true)
 plt = plot(x[1,:], ud[1,:,begin], w = 4, c = :black, label = nothing)
 plot!(plt, x[1,:], ud[1,:,end  ], w = 4, c = :black, label = "data")
 plot!(plt, x[1,:], up[1,:,end  ], w = 4, c = :red  , label = "pred")
 display(plt)
+
+@show sqrt(mse(up, ud) / mse(ud, 0 * ud))
+@show norm(up - ud, Inf) / sqrt(mse(ud, 0 * ud))
 #======================================================#
 nothing
 

examples/burgers_fourier1D/autodecode1.jl

@@ -0,0 +1,51 @@
+#
+using GeometryLearning
+include(joinpath(pkgdir(GeometryLearning), "examples", "autodecoder.jl"))
+#======================================================#
+
+rng = Random.default_rng()
+Random.seed!(rng, 460)
+
+prob = BurgersViscous1D(1f-4)
+device = Lux.gpu_device()
+datafile = joinpath(@__DIR__, "burg_visc_re10k", "data.jld2")
+modeldir = joinpath(@__DIR__, "dump1")
+modelfile = joinpath(modeldir, "model_08.jld2")
+
+E = 3500  # epochs
+l = 8     # latent
+h = 5     # num hidden
+w = 128   # width
+
+λ1, λ2 = 0f0, 0f0
+σ2inv, α = 1f-1, 0f-0 # 1f-1, 1f-3
+weight_decays = 1f-2  # 1f-2
+WeightDecayOpt = IdxWeightDecay
+weight_decay_ifunc = decoder_indices
+
+_Ib, Ib_ = [5,], [5,] # 1, 2, 3 work
+Ix  = Colon() # LinRange(1, 8192, 1024) .|> Base.Fix1(round, Int)
+_It = Colon() # LinRange(1, 1000, 200 ) .|> Base.Fix1(round, Int)
+makedata_kws = (; Ix, _Ib, Ib_, _It = _It, It_ = :)
+
+## train
+isdir(modeldir) && rm(modeldir, recursive = true)
+train_SNF(datafile, modeldir, l, h, w, E;
+    rng, warmup = true, makedata_kws,
+    λ1, λ2, σ2inv, α, weight_decays, device,
+    WeightDecayOpt, weight_decay_ifunc,
+)
+
+outdir = joinpath(modeldir, "results")
+postprocess_SNF(prob, datafile, modelfile, outdir; rng, device,
+    makeplot = true, verbose = true)
+x, t, ud, up, _ = evolve_SNF(prob, datafile, modelfile, Ib_[1]; rng, device, verbose = true)
+plt = plot(x[1,:], ud[1,:,begin], w = 4, c = :black, label = nothing)
+plot!(plt, x[1,:], ud[1,:,end  ], w = 4, c = :black, label = "data")
+plot!(plt, x[1,:], up[1,:,end  ], w = 4, c = :red  , label = "pred")
+display(plt)
+
+@show sqrt(mse(up, ud) / mse(ud, 0 * ud))
+@show norm(up - ud, Inf) / sqrt(mse(ud, 0 * ud))
+#======================================================#
+nothing

examples/burgers_fourier1D/data_burg1d/README

@@ -0,0 +1 @@
+mu = (0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2)

examples/ks_fourier/autodecode.jl

@@ -4,26 +4,24 @@ include(joinpath(pkgdir(GeometryLearning), "examples", "autodecoder.jl"))
 #======================================================#
 
 rng = Random.default_rng()
-Random.seed!(rng, 123)
+Random.seed!(rng, 213)
 
+prob = KuramotoSivashinsky1D(0.01f0)
 device = Lux.gpu_device()
 datafile = joinpath(@__DIR__, "data_ks/", "data.jld2")
 modeldir = joinpath(@__DIR__, "dump")
 modelfile = joinpath(modeldir, "model_08.jld2")
 
-prob = KuramotoSivashinsky1D(0.01f0)
-
-E = 2100  # epochs
+E = 3500  # epochs
 l = 16    # latent
 h = 5     # num hidden
 w = 128   # width
 
-### WORKS
 λ1, λ2 = 0f0, 0f0
 σ2inv, α = 1f-1, 0f-0 # 1f-1, 1f-3
 weight_decays = 1f-2  # 1f-2
-WeightDecayOpt = DecoderWeightDecay
-weight_decay_ifunc = nothing
+WeightDecayOpt = IdxWeightDecay
+weight_decay_ifunc = decoder_W_indices
 
 ## train
 isdir(modeldir) && rm(modeldir, recursive = true)