From e612abb172da9944301e3e9b4826ea8427bcc570 Mon Sep 17 00:00:00 2001 From: Thibault Neveu Date: Tue, 19 Jun 2018 18:55:06 +0100 Subject: [PATCH 1/9] WIP: Implement DDPG --- demo/webapp/level2.html | 8 +- demo/webapp/public/js/DDPG/ddpg.js | 40 ++++ .../public/js/{level2.js => DDPG/index.js} | 3 + demo/webapp/public/js/DDPG/memory.js | 77 ++++++++ demo/webapp/public/js/DDPG/models.js | 185 ++++++++++++++++++ demo/webapp/public/js/DDPG/noise.js | 41 ++++ 6 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 demo/webapp/public/js/DDPG/ddpg.js rename demo/webapp/public/js/{level2.js => DDPG/index.js} (89%) create mode 100644 demo/webapp/public/js/DDPG/memory.js create mode 100644 demo/webapp/public/js/DDPG/models.js create mode 100644 demo/webapp/public/js/DDPG/noise.js diff --git a/demo/webapp/level2.html b/demo/webapp/level2.html index 78bf318..3d30368 100644 --- a/demo/webapp/level2.html +++ b/demo/webapp/level2.html @@ -37,11 +37,17 @@

Current state (Lidar points)



+ - + + + + + + diff --git a/demo/webapp/public/js/DDPG/ddpg.js b/demo/webapp/public/js/DDPG/ddpg.js new file mode 100644 index 0000000..1fe09dc --- /dev/null +++ b/demo/webapp/public/js/DDPG/ddpg.js @@ -0,0 +1,40 @@ +// This class is called from js/DDPG/index.js +class DDPG { + + constructor(){ + // Default Config + this.config = { + "stateSize": 25, + "nbActions": 2, + "layerNom": true, + "normalizeObservations": true, + "seed": 0, + "criticL2Reg": 0.01, + "batchSize": 64, + "actorLr": 0.0001, + "criticLr": 0.001, + "gamma": 0.99, + "rewardScale": 1, + "nbEpochs": 500, + "nbEpochsCycle": 800, + "nbTrainSteps": 50, + "nbRolloutStep": 100 + }; + // Inputs + const obsInput = tf.input({shape: [this.config.stateSize]}); + const actionInput = tf.input({shape: [this.config.nbActions]}); + + // From js/DDPG/noise.js + this.paramNoise = new AdaptiveParamNoiseSpec(); + // Buffer replay + // The baseline use 1e6 but this size should be enough + this.memory = new Memory(1000); + // Actor and Critic are from js/DDPG/models.js + this.actor = new Actor( + this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed); + this.critic = new Critic( + this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed); + this.actor.buildModel(obsInput); + this.critic.buildModel(obsInput, actionInput); + } +}; \ No newline at end of file diff --git a/demo/webapp/public/js/level2.js b/demo/webapp/public/js/DDPG/index.js similarity index 89% rename from demo/webapp/public/js/level2.js rename to demo/webapp/public/js/DDPG/index.js index 5b1b5b7..30ebda4 100644 --- a/demo/webapp/public/js/level2.js +++ b/demo/webapp/public/js/DDPG/index.js @@ -1,5 +1,8 @@ let levelUrl = metacar.level.level2; +// js/DDPG/ddpg.js +var ddpg = new DDPG(); + var env = new metacar.env("canvas", levelUrl); env.setAgentMotion(metacar.motion.ControlMotion, {}); diff --git a/demo/webapp/public/js/DDPG/memory.js b/demo/webapp/public/js/DDPG/memory.js new file mode 100644 index 0000000..6b13d58 --- /dev/null +++ b/demo/webapp/public/js/DDPG/memory.js @@ -0,0 +1,77 @@ + +class Memory { + + /** + * @param maxlen (number) Buffer limit + */ + constructor(maxlen){ + this.maxlen = maxlen; + this.length = 0; + this.start = 0; + + this.obs0List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); + this.obs1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); + this.rewardsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); + this.actionsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); + this.terminals1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); + } + + /** + * @param idx (number) + */ + getItem(idx){ + if (idx < 0 || idx >= this.length){ + console.error("Memory.getItem: idx not in range."); + } + return this.data[(this.start + idx) % this.maxlen] + } + + /** + * Sample a batch + * @param batchSize (number) + * @return batch [] + */ + getBatch(batchSize){ + const arrLength = this.obs0List.length; + const batch = { + 'obs0': [], + 'obs1': [], + 'rewards': [], + 'actions': [], + 'terminals1': [], + }; + for (let b=0; b < batchSize; b++){ + let id = Math.floor(Math.random() * arrLength); + batch.obs0.push(this.obs0List[id]); + batch.obs1.push(this.obs1List[id]); + batch.rewards.push(this.rewardsList[id]); + batch.actions.push(this.actionsList[id]); + batch.terminals1.push(this.terminals1List[id]); + } + return batch + } + + /** + * @param obs0 [] + * @param action (number) + * @param reward (number) + * @param obs1 [] + * @param terminal1 (boolean) + */ + append(obs0, action, reward, obs1, terminal1){ + if (this.length < this.maxlen){ + this.length += 1; + } + else if (this.length == this.maxlen) { + this.start = (this.start + 1) % this.maxlen; + } + else { + console.error("Memory.append: This should never be printed"); + } + this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0; + this.obs1List[(this.start + this.length - 1) % this.maxlen] = action; + this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward; + this.actionsList[(this.start + this.length - 1) % this.maxlen] = obs1; + this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1; + } +} \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/models.js b/demo/webapp/public/js/DDPG/models.js new file mode 100644 index 0000000..5a97256 --- /dev/null +++ b/demo/webapp/public/js/DDPG/models.js @@ -0,0 +1,185 @@ + +class Actor{ + + /** + * @param stateSize(number) + * @param nbActions (number) + * @param layerNorm (boolean) + * @param seed (number) + */ + constructor(stateSize, nbActions, layerNorm, seed) { + this.stateSize = stateSize; + this.nbActions = nbActions; + this.layerNorm = layerNorm; + this.seed = seed; + } + + /** + * + * @param obs tf.input + */ + buildModel(obs){ + this.firstLayerBatchNorm = null; + this.secondLayerBatchNorm = null; + + this.relu = tf.layers.thresholdedReLU(); + + // First layer with BatchNormalization + this.firstLayer = tf.layers.dense({ + inputShape: this.stateSize, + units: 64, + kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), + activation: 'linear', // relu is add later + useBias: true, + biasInitializer: "zeros" + }); + if (this.layerNorm){ + // WARNING: BatchNormalization instead of layerNormalization + this.firstLayerBatchNorm = tf.layers.batchNormalization({ + scale: true, + center: true + }); + } + + // Second layer with BatchNormalization + this.secondLayer = tf.layers.dense({ + inputShape: 64, + units: 64, + kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), + activation: 'linear', // relu is add later + useBias: true, + biasInitializer: "zeros" + }); + if (this.layerNorm){ + // WARNING: BatchNormalization instead of layerNormalization + this.secondLayerBatchNorm = tf.layers.batchNormalization({ + scale: true, + center: true + }); + } + + // Ouput layer + this.outputLayer = tf.layers.dense({ + inputShape: 64, + units: this.nbActions, + kernelInitializer: tf.initializers.randomUniform({ + minval: 0.003, maxval: 0.003, seed: this.seed}), + activation: 'tanh', + useBias: true, + biasInitializer: "zeros" + }); + + // Actor prediction + const predict = () => { + return tf.tidy(() => { + let l1 = this.firstLayer.apply(obs); + if (this.firstLayerBatchNorm){ + l1 = this.firstLayerBatchNorm.apply(l1); + } + //l1 = this.relu.apply(l1); + let l2 = this.secondLayer.apply(l1); + if (this.secondLayerBatchNorm){ + l2 = this.secondLayerBatchNorm.apply(l2); + } + //l2 = this.relu.apply(l2); + return this.outputLayer.apply(l2); + }); + } + const output = predict(); + this.model = tf.model({inputs: this.obs, outputs: output}); + } + +}; + +class Critic { + + /** + * @param stateSize(number) + * @param nbActions (number) + * @param layerNorm (boolean) + * @param seed (number) + */ + constructor(stateSize, nbActions, layerNorm, seed) { + this.stateSize = stateSize; + this.nbActions = nbActions; + this.layerNorm = layerNorm; + } + + /** + * + * @param obs tf.input + * @param action tf.input + */ + buildModel(obs, action){ + this.firstLayerBatchNorm = null; + this.secondLayerBatchNorm = null; + + this.relu = tf.layers.thresholdedReLU(); + + // First layer with BatchNormalization + this.firstLayer = tf.layers.dense({ + inputShape: this.stateSize, + units: 64, + kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), + activation: 'linear', // relu is add later + useBias: true, + biasInitializer: "zeros" + }); + if (this.layerNorm){ + // WARNING: BatchNormalization instead of layerNormalization + this.firstLayerBatchNorm = tf.layers.batchNormalization({ + scale: true, + center: true + }); + } + + // Second layer with BatchNormalization + this.secondLayer = tf.layers.dense({ + inputShape: 64 + this.nbActions, // Previous layer + action + units: 64, + kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), + activation: 'linear', // relu is add later + useBias: true, + biasInitializer: "zeros" + }); + if (this.layerNorm){ + // WARNING: BatchNormalization instead of layerNormalization + this.secondLayerBatchNorm = tf.layers.batchNormalization({ + scale: true, + center: true + }); + } + + // Ouput layer + this.outputLayer = tf.layers.dense({ + inputShape: 64, + units: 1, + kernelInitializer: tf.initializers.randomUniform({ + minval: 0.003, maxval: 0.003, seed: this.seed}), + activation: 'tanh', + useBias: true, + biasInitializer: "zeros" + }); + + // Actor prediction + const predict = () => { + return tf.tidy(() => { + let l1 = this.firstLayer.apply(obs); + l1 = l1.concat(action); + if (this.firstLayerBatchNorm){ + l1 = this.firstLayerBatchNorm.apply(l1); + } + //l1 = this.relu(l1); + let l2 = this.secondLayer.apply(l1); + if (this.secondLayerBatchNorm){ + l2 = this.secondLayerBatchNorm.apply(l2); + } + //l2 = this.relu.apply(l2); + return this.outputLayer.apply(l2); + }); + } + const output = predict(); + this.model = tf.model({inputs: this.obs, outputs: output}); + } + +}; diff --git a/demo/webapp/public/js/DDPG/noise.js b/demo/webapp/public/js/DDPG/noise.js new file mode 100644 index 0000000..c173625 --- /dev/null +++ b/demo/webapp/public/js/DDPG/noise.js @@ -0,0 +1,41 @@ +/** + * Noise class + * The original baseline is made of three noise + * AdaptiveParamNoiseSpec, ActionNoise and NormalActionNoise + * Only AdaptiveParamNoiseSpec is implemented for now + * See "C Adapative Scaling" Page 14 in the paper. + */ + +class AdaptiveParamNoiseSpec{ + + /** + * @param conf Object + * conf.initialStddev: 0.1 default // σ + * conf.desiredActionStddev: 0.1 default // δ + * conf.adoptionCoefficient: 1.01 default // α + */ + constructor(conf){ + conf = conf || {}; + this.initialStddev = conf.initialStddev || 0.1; + this.desiredActionStddev = conf.initialStddev || 0.1; + this.adoptionCoefficient = conf.adoptionCoefficient || 1.01; + this.currentStddev = conf.initialStddev; + } + + /** + * The distance from the Adaptive scaling + * @param distance number + */ + adapt(distance){ + // if d(π, _π_) > δ then σ = σ/α + if (distance > this.desiredActionStddev){ + // Decrease σ + this.currentStddev /= this.adoptionCoefficient; + } + else{ + // σ = σ*α + // Increase σ + this.currentStddev *= this.adoptionCoefficient; + } + } +}; \ No newline at end of file From dac1db06ef7e1484a3841f528d848211eec9325d Mon Sep 17 00:00:00 2001 From: Thibault Neveu Date: Fri, 22 Jun 2018 19:36:29 +0100 Subject: [PATCH 2/9] DDPG Implemented --- demo/webapp/level2.html | 5 + demo/webapp/public/js/DDPG/ddpg.js | 279 ++++++++++++++++++++--- demo/webapp/public/js/DDPG/ddpg_agent.js | 168 ++++++++++++++ demo/webapp/public/js/DDPG/index.js | 36 ++- demo/webapp/public/js/DDPG/memory.js | 16 +- demo/webapp/public/js/DDPG/models.js | 172 ++++++++++---- demo/webapp/public/js/DDPG/noise.js | 4 +- demo/webapp/public/js/viewer.js | 47 ++++ src/car.ts | 38 ++- src/control_motion_engine.ts | 12 +- src/level.ts | 13 +- src/metacar.ts | 16 +- src/ui_event.ts | 1 + src/world.ts | 9 + 14 files changed, 702 insertions(+), 114 deletions(-) create mode 100644 demo/webapp/public/js/DDPG/ddpg_agent.js diff --git a/demo/webapp/level2.html b/demo/webapp/level2.html index 3d30368..d0c93ee 100644 --- a/demo/webapp/level2.html +++ b/demo/webapp/level2.html @@ -37,6 +37,10 @@

Current state (Lidar points)



+
+ + + @@ -48,6 +52,7 @@

Current state (Lidar points)



+ diff --git a/demo/webapp/public/js/DDPG/ddpg.js b/demo/webapp/public/js/DDPG/ddpg.js index 1fe09dc..04e95f9 100644 --- a/demo/webapp/public/js/DDPG/ddpg.js +++ b/demo/webapp/public/js/DDPG/ddpg.js @@ -1,40 +1,251 @@ -// This class is called from js/DDPG/index.js + +function logTfMemory(){ + let mem = tf.memory(); + console.log("numBytes:" + mem.numBytes + + "\nnumBytesInGPU:" + mem.numBytesInGPU + + "\nnumDataBuffers:" + mem.numDataBuffers + + "\nnumTensors:" + mem.numTensors); +} + +// This class is called from js/DDPG/ddpg_agent.js class DDPG { - constructor(){ - // Default Config - this.config = { - "stateSize": 25, - "nbActions": 2, - "layerNom": true, - "normalizeObservations": true, - "seed": 0, - "criticL2Reg": 0.01, - "batchSize": 64, - "actorLr": 0.0001, - "criticLr": 0.001, - "gamma": 0.99, - "rewardScale": 1, - "nbEpochs": 500, - "nbEpochsCycle": 800, - "nbTrainSteps": 50, - "nbRolloutStep": 100 - }; + /** + * @param config (Object) + * @param actor (Actor class) + * @param critic (Critic class) + * @param memory (Memory class) + * @param noise (Noise class) + */ + constructor(actor, critic, memoryPos, memoryNeg, noise, config){ + this.actor = actor; + this.critic = critic; + this.memoryPos = memoryPos; + this.memoryNeg = memoryNeg; + this.noise = noise; + this.config = config; + this.tfGamma = tf.scalar(config.gamma); + // Inputs - const obsInput = tf.input({shape: [this.config.stateSize]}); - const actionInput = tf.input({shape: [this.config.nbActions]}); - - // From js/DDPG/noise.js - this.paramNoise = new AdaptiveParamNoiseSpec(); - // Buffer replay - // The baseline use 1e6 but this size should be enough - this.memory = new Memory(1000); - // Actor and Critic are from js/DDPG/models.js - this.actor = new Actor( - this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed); - this.critic = new Critic( - this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed); + let obsInput = tf.input({batchShape: [null, this.config.stateSize]}); + let actionInput = tf.input({batchShape: [null, this.config.nbActions]}); + + if (config.normalizeObservations){ + tf.layers.batchNormalization({ + scale: true, + center: true + }).apply(obsInput); + } + + // Randomly Initialize actor network μ(s) this.actor.buildModel(obsInput); + // Randomly Initialize critic network Q(s, a) this.critic.buildModel(obsInput, actionInput); + + // Define in js/DDPG/models.js + // Init target network Q' and μ' with the same weights + this.actorTarget = copyModel(this.actor, Actor); + this.criticTarget = copyModel(this.critic, Critic); + // Perturbed Actor (See parameter space noise Exploration paper) + this.perturbedActor = copyModel(this.actor, Actor); + //this.adaptivePerturbedActor = copyModel(this.actor, Actor); + + this.actorOptimiser = tf.train.adam(this.config.actorLr); + this.criticOptimiser = tf.train.adam(this.config.criticLr); + + this.criticWeights = []; + for (let w = 0; w < this.critic.model.weights.length; w++){ + this.criticWeights.push(this.critic.model.weights[w].val); + } + + this.actorWeights = []; + for (let w = 0; w < this.actor.model.weights.length; w++){ + this.actorWeights.push(this.actor.model.weights[w].val); + } + + // Return a batch from positive and negative experiences + this.memory = { + getBatch: (size) => { + let batch1 = this.memoryPos.getBatch(size/2); + let batch2 = this.memoryNeg.getBatch(size/2); + return { + 'obs0': batch1.obs0.concat(batch2.obs0), + 'obs1': batch1.obs1.concat(batch2.obs1), + 'rewards': batch1.rewards.concat(batch2.rewards), + 'actions': batch1.actions.concat(batch2.actions), + 'terminals': batch1.terminals.concat(batch2.terminals), + }; + } + }; + + assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev); + } + + + /** + * Distance Measure for DDPG + * See parameter space noise Exploration paper + * obs (Tensor2d) Observations + */ + distanceMeasure(observations) { + return tf.tidy(() => { + const pertubedPredictions = this.perturbedActor.model.predict(observations); + const predictions = this.actor.model.predict(observations); + + const distance = tf.square(pertubedPredictions.sub(predictions)).mean().sqrt(); + return distance; + }); + } + + /** + * AdaptParamNoise + */ + adaptParamNoise(){ + const batch = this.memory.getBatch(this.config.batchSize); + const tfObs0 = tf.tensor2d(batch.obs0); + const distance = this.distanceMeasure(tfObs0); + + assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev); + + const distanceV = distance.buffer().values; + this.noise.adapt(distanceV[0]); + setMetric("Distance", distanceV[0]) + + distance.dispose(); + tfObs0.dispose(); + } + + /** + * Eval the actions and the Q function + * @param states (tf.tensor2d) + * @return actions and qValues + */ + eval(observation){ + const tfActions = this.perturbedActor.model.predict(observation); + const tfQValues = this.critic.model.predict([observation, tfActions]); + return {tfActions, tfQValues}; + } + + /** + * Get the estimation of the Q value given the state + * and the action + * @param state number[] + * @param action [a, steering] + */ + getQvalue(state, a){ + const st = tf.tensor2d([state]); + const tfa = tf.tensor2d([a]); + const q = this.critic.model.predict([st, tfa]); + const v = q.buffer().values + st.dispose(); + tfa.dispose(); + q.dispose(); + return v[0]; + } + + /** + * @param observation (tf.tensor2d) + * @return (tf.tensor1d) + */ + predict(observation){ + const tfActions = this.actor.model.predict(observation); + return tfActions; + } + + /** + * @param observation (tf.tensor2d) + * @return (tf.tensor1d) + */ + perturbedPrediction(observation){ + const tfActions = this.perturbedActor.model.predict(observation); + return tfActions; + } + + /** + * Update the two target network + */ + targetUpdate(){ + // Define in js/DDPG/models.js + //assignModel(this.critic, this.criticTarget); + //assignModel(this.actor, this.actorTarget); + targetUpdate(this.criticTarget, this.critic, this.config); + targetUpdate(this.actorTarget, this.actor, this.config); + } + + trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals){ + + const criticLoss = this.criticOptimiser.minimize(() => { + const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]); + + const tfAPredictions = this.actorTarget.model.predict(tfObs1); + const tfQPredictions = this.criticTarget.model.predict([tfObs1, tfAPredictions]); + + const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions)); + + const loss = tf.sub(tfQTargets, tfQPredictions0).square().mean(); + return loss; + }, true, this.criticWeights); + + setMetric("CriticLoss", criticLoss.buffer().values[0]); + criticLoss.dispose(); + } + + trainActor(tfObs0, it){ + for (let i = 0; i < it; i++){ + const actorLoss = this.actorOptimiser.minimize(() => { + const tfAPredictions0 = this.actor.model.predict(tfObs0); + const tfQPredictions0 = this.critic.model.predict([tfObs0, tfAPredictions0]); + const loss = tf.mean(tfQPredictions0).mul(tf.scalar(-1)); + return loss; + }, true, this.actorWeights); + const vLoss = actorLoss.buffer().values[0]; + setMetric("ActorLoss", actorLoss.buffer().values[0]); + actorLoss.dispose(); + } + } + + getTfBatch(){ + // Get batch + const batch = this.memory.getBatch(this.config.batchSize); + // Convert to tensors + const tfActions = tf.tensor2d(batch.actions); + const tfObs0 = tf.tensor2d(batch.obs0); + const tfObs1 = tf.tensor2d(batch.obs1); + const _tfRewards = tf.tensor1d(batch.rewards); + const _tfTerminals = tf.tensor1d(batch.terminals); + + const tfRewards = _tfRewards.expandDims(1); + const tfTerminals = _tfTerminals.expandDims(1); + + _tfRewards.dispose(); + _tfTerminals.dispose(); + + return { + tfActions, tfObs0, tfObs1, tfRewards, tfTerminals + } + } + + async optimizeCritic(){ + const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + + this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); + + tfActions.dispose(); + tfObs0.dispose(); + tfObs1.dispose(); + tfRewards.dispose(); + tfTerminals.dispose(); + } + + async optimizeActor(it=1){ + const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + + this.trainActor(tfObs0, it); + + tfActions.dispose(); + tfObs0.dispose(); + tfObs1.dispose(); + tfRewards.dispose(); + tfTerminals.dispose(); } -}; \ No newline at end of file +} \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/ddpg_agent.js b/demo/webapp/public/js/DDPG/ddpg_agent.js new file mode 100644 index 0000000..add5eaf --- /dev/null +++ b/demo/webapp/public/js/DDPG/ddpg_agent.js @@ -0,0 +1,168 @@ + +// This class is called from js/DDPG/index.js +class DDPGAgent { + + /** + * @param env (metacar.env) Set in js/DDPG/index.js + */ + constructor(env){ + this.stopTraining = false; + this.env = env; + // Default Config + this.config = { + "stateSize": 26, + "nbActions": 2, + "layerNom": true, + "normalizeObservations": true, + "seed": 0, + "criticL2Reg": 0.01, + "batchSize": 32, + "actorLr": 0.0001, + "criticLr": 0.001, + "gamma": 0.99, + "rewardScale": 1, + "nbEpochs": 500, + "nbEpochsCycle": 100, + "nbTrainSteps": 50, + "tau": 0.001, + "paramNoiseAdaptionInterval": 50, + }; + // From js/DDPG/noise.js + this.noise = new AdaptiveParamNoiseSpec(); + + // Configure components. + + // Buffer replay + // The baseline use 1e6 but this size should be enough for this problem + this.memoryPos = new Memory(5000); + this.memoryNeg = new Memory(5000); + // Actor and Critic are from js/DDPG/models.js + this.actor = new Actor(this.config); + this.critic = new Critic(this.config); + + // Seed javascript + Math.seedrandom(0); + + this.rewardsList = []; + + // DDPG + this.ddpg = new DDPG(this.actor, this.critic, this.memoryPos, this.memoryNeg, this.noise, this.config); + } + + /** + * Play one step + */ + play(){ + // Get the current state + const state = agent.env.getState().linear; + // Pick an action + const tfActions = agent.ddpg.predict(tf.tensor2d([state])); + const actions = tfActions.buffer().values; + agent.env.step([actions[0], actions[1]]); + tfActions.dispose(); + } + + /** + * Get the estimation of the Q value given the state + * and the action + * @param state number[] + * @param action [a, steering] + */ + getQvalue(state, a){ + return this.ddpg.getQvalue(state, a); + } + + stop(){ + this.stopTraining = true; + } + + /** + * Step into the training environement + * @param tfPreviousStep (tf.tensor2d) Current state + * @param mPreviousStep number[] + * @return {done, state} One boolean and the new state + */ + stepTrain(tfPreviousStep, mPreviousStep){ + // Get actions + const tfActions = this.ddpg.perturbedPrediction(tfPreviousStep); + // Step in the environment with theses actions + let mAcions = tfActions.buffer().values; + let mReward = this.env.step([mAcions[0], mAcions[1]]); + this.rewardsList.push(mReward); + // Get the new observations + let mState = this.env.getState().linear; + let tfState = tf.tensor2d([mState]); + let mDone = 0; + if (mReward == -1){ + mDone = 1; + } + + // Add the new tuple to the buffer + if (mReward >= 0) + this.ddpg.memoryPos.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); + else + this.ddpg.memoryNeg.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); + + // Dispose tensor + tfPreviousStep.dispose(); + tfActions.dispose(); + return {mDone, mState, tfState} + } + + /** + * Train DDPG Agent + */ + async train(realTime){ + this.stopTraining = false; + // One epoch + for (let e=0; e < this.config.nbEpochs; e++){ + // Perform cycles. + for (let c=0; c < this.config.nbEpochsCycle; c++){ + if (c%10==0){ + logTfMemory(); + } + this.rewardsList = []; + // Perform rollouts. + // Get current observation + let mPreviousStep = this.env.getState().linear; + let tfPreviousStep = tf.tensor2d([mPreviousStep]); + let step = 0; + console.time("LoopTime"); + for (step=0; step < 800; step++){ + let rel = this.stepTrain(tfPreviousStep, mPreviousStep); + mPreviousStep = rel.mState; + tfPreviousStep = rel.tfState; + if (rel.mDone){ + break; + } + if (this.stopTraining){ + this.env.render(true); + return; + } + if (realTime) + await tf.nextFrame(); + } + console.timeEnd("LoopTime"); + this.env.reset(); + tfPreviousStep.dispose(); + // Mean is define is js/utils.js + console.log("e="+ e +", c="+c); + setMetric("Reward", mean(this.rewardsList)); + setMetric("EpisodeDuration", step); + this.ddpg.adaptParamNoise(); + await tf.nextFrame(); + } + console.time("LoopTrain"); + for (let t=0; t < 100; t++){ + this.ddpg.optimizeCritic(); + } + for (let t=0; t < 100; t++){ + this.ddpg.optimizeActor(); + } + console.timeEnd("LoopTrain"); + this.ddpg.targetUpdate(); + } + this.env.render(true); + } + +}; \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/index.js b/demo/webapp/public/js/DDPG/index.js index 30ebda4..5ceb838 100644 --- a/demo/webapp/public/js/DDPG/index.js +++ b/demo/webapp/public/js/DDPG/index.js @@ -1,17 +1,41 @@ let levelUrl = metacar.level.level2; -// js/DDPG/ddpg.js -var ddpg = new DDPG(); - var env = new metacar.env("canvas", levelUrl); env.setAgentMotion(metacar.motion.ControlMotion, {}); + +// js/DDPG/ddpg.js +var agent = new DDPGAgent(env); + +initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "Distance"]); + env.loop(() => { let state = env.getState(); - displayState("realtime_viewer", state, 200, 200); + displayState("realtime_viewer", state.lidar, 200, 200); let reward = env.getLastReward(); - displayScores("realtime_viewer", [], reward, []); + const qValue = agent.getQvalue(state.linear, [state.a, state.steering]); + displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]); }); -env.load(); \ No newline at end of file +env.load().then(() => { + // Train agent + env.addEvent("train", () => { + agent.train(false); + }); + + env.addEvent("play", () => { + agent.play(); + }); + + env.addEvent("TrainRealTime", () => { + env.steping(false); + agent.train(true); + }); + + env.addEvent("stop", () => { + agent.stop(); + }); + + env.addEvent("reset_env"); +}); diff --git a/demo/webapp/public/js/DDPG/memory.js b/demo/webapp/public/js/DDPG/memory.js index 6b13d58..fc8db0a 100644 --- a/demo/webapp/public/js/DDPG/memory.js +++ b/demo/webapp/public/js/DDPG/memory.js @@ -32,21 +32,24 @@ class Memory { * @return batch [] */ getBatch(batchSize){ - const arrLength = this.obs0List.length; + const arrLength = this.length; const batch = { 'obs0': [], 'obs1': [], 'rewards': [], 'actions': [], - 'terminals1': [], + 'terminals': [], }; + if (batchSize > this.length){ + return batch; + } for (let b=0; b < batchSize; b++){ let id = Math.floor(Math.random() * arrLength); batch.obs0.push(this.obs0List[id]); batch.obs1.push(this.obs1List[id]); batch.rewards.push(this.rewardsList[id]); batch.actions.push(this.actionsList[id]); - batch.terminals1.push(this.terminals1List[id]); + batch.terminals.push(this.terminals1List[id]); } return batch } @@ -63,15 +66,18 @@ class Memory { this.length += 1; } else if (this.length == this.maxlen) { + //this.obs0List[(this.start + this.length - 1) % this.maxlen].dispose(); + //this.obs1List[(this.start + this.length - 1) % this.maxlen].dispose(); + //this.actionsList[(this.start + this.length - 1) % this.maxlen].dispose(); this.start = (this.start + 1) % this.maxlen; } else { console.error("Memory.append: This should never be printed"); } this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0; - this.obs1List[(this.start + this.length - 1) % this.maxlen] = action; + this.obs1List[(this.start + this.length - 1) % this.maxlen] = obs1; this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward; - this.actionsList[(this.start + this.length - 1) % this.maxlen] = obs1; + this.actionsList[(this.start + this.length - 1) % this.maxlen] = action; this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1; } } \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/models.js b/demo/webapp/public/js/DDPG/models.js index 5a97256..b47a55d 100644 --- a/demo/webapp/public/js/DDPG/models.js +++ b/demo/webapp/public/js/DDPG/models.js @@ -1,17 +1,93 @@ +/** + * Copy a model + * @param model Actor|Critic instance + * @param instance Actor|Critic + * @return Copy of the model + */ +function copyModel(model, instance){ + return tf.tidy(() => { + nModel = new instance(model.config); + // action might be not required + nModel.buildModel(model.obs, model.action); + const weights = model.model.weights; + for (let m=0; m < weights.length; m++){ + nModel.model.weights[m].val.assign(weights[m].val); + } + return nModel; + }) +} + +/** + * Usefull method to copy a model + * @param model Actor + * @param perturbedActor Actor + * @param stddev (number) + * @return Copy of the model + */ +function assignAndStd(actor, perturbedActor, stddev){ + return tf.tidy(() => { + const weights = actor.model.weights; + for (let m=0; m < weights.length; m++){ + let shape = perturbedActor.model.weights[m].val.shape; + let randomTensor = tf.randomNormal(shape, 0, stddev); + let nValue = weights[m].val.add(randomTensor); + perturbedActor.model.weights[m].val.assign(nValue); + } + }); +} + +/** + * Usefull method to copy a model + * @param model Actor + * @param perturbedActor Actor + * @return Copy of the model + */ +function assignModel(model, targetModel){ + return tf.tidy(() => { + const weights = model.model.weights; + for (let m=0; m < weights.length; m++){ + let nValue = weights[m].val; + targetModel.model.weights[m].val.assign(nValue); + } + }); +} + + +/** + * Usefull method to copy a model + * @param target Actor|Critic + * @param perturbedActor Actor|Critic + * @param config (Object) + * @return Copy of the model + */ +function targetUpdate(target, original, config){ + return tf.tidy(() => { + const originalW = original.model.weights; + const targetW = target.model.weights; + + const one = tf.scalar(1); + const tau = tf.scalar(config.tau); + + for (let m=0; m < originalW.length; m++){ + let nValue = tau.mul(originalW[m].val).add(targetW[m].val.mul(one.sub(tau))); + target.model.weights[m].val.assign(nValue); + } + }); +} + class Actor{ /** - * @param stateSize(number) - * @param nbActions (number) - * @param layerNorm (boolean) - * @param seed (number) + @param config (Object) */ - constructor(stateSize, nbActions, layerNorm, seed) { - this.stateSize = stateSize; - this.nbActions = nbActions; - this.layerNorm = layerNorm; - this.seed = seed; + constructor(config) { + this.stateSize = config.stateSize; + this.nbActions = config.nbActions; + this.layerNorm = config.layerNorm; + this.seed = config.seed; + this.config = config; + this.obs = null; } /** @@ -19,14 +95,17 @@ class Actor{ * @param obs tf.input */ buildModel(obs){ + this.obs = obs; + + this.firstLayerBatchNorm = null; this.secondLayerBatchNorm = null; - this.relu = tf.layers.thresholdedReLU(); + this.relu1 = tf.layers.activation({activation: 'relu'}); + //this.relu2 = tf.layers.activation({activation: 'relu'}); // First layer with BatchNormalization this.firstLayer = tf.layers.dense({ - inputShape: this.stateSize, units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later @@ -40,10 +119,10 @@ class Actor{ center: true }); } - + + /* // Second layer with BatchNormalization this.secondLayer = tf.layers.dense({ - inputShape: 64, units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later @@ -56,11 +135,10 @@ class Actor{ scale: true, center: true }); - } + }*/ // Ouput layer this.outputLayer = tf.layers.dense({ - inputShape: 64, units: this.nbActions, kernelInitializer: tf.initializers.randomUniform({ minval: 0.003, maxval: 0.003, seed: this.seed}), @@ -68,41 +146,43 @@ class Actor{ useBias: true, biasInitializer: "zeros" }); - - // Actor prediction - const predict = () => { + + this.predict = () => { return tf.tidy(() => { let l1 = this.firstLayer.apply(obs); if (this.firstLayerBatchNorm){ l1 = this.firstLayerBatchNorm.apply(l1); } - //l1 = this.relu.apply(l1); + l1 = this.relu1.apply(l1); + /* let l2 = this.secondLayer.apply(l1); if (this.secondLayerBatchNorm){ l2 = this.secondLayerBatchNorm.apply(l2); } - //l2 = this.relu.apply(l2); - return this.outputLayer.apply(l2); + l2 = this.relu2.apply(l2); + */ + + return this.outputLayer.apply(l1); }); } - const output = predict(); - this.model = tf.model({inputs: this.obs, outputs: output}); + const output = this.predict(); + this.model = tf.model({inputs: obs, outputs: output}); } - }; class Critic { /** - * @param stateSize(number) - * @param nbActions (number) - * @param layerNorm (boolean) - * @param seed (number) + * @param config (Object) */ - constructor(stateSize, nbActions, layerNorm, seed) { - this.stateSize = stateSize; - this.nbActions = nbActions; - this.layerNorm = layerNorm; + constructor(config) { + this.stateSize = config.stateSize; + this.nbActions = config.nbActions; + this.layerNorm = config.layerNorm; + this.seed = config.seed; + this.config = config; + this.obs = null; + this.action = null; } /** @@ -111,14 +191,18 @@ class Critic { * @param action tf.input */ buildModel(obs, action){ + this.obs = obs; + this.action = action; + this.firstLayerBatchNorm = null; this.secondLayerBatchNorm = null; - this.relu = tf.layers.thresholdedReLU(); + this.relu1 = tf.layers.activation({activation: 'relu'}); + this.relu2 = tf.layers.activation({activation: 'relu'}); + this.concat = tf.layers.concatenate(); // First layer with BatchNormalization this.firstLayer = tf.layers.dense({ - inputShape: this.stateSize, units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later @@ -135,7 +219,7 @@ class Critic { // Second layer with BatchNormalization this.secondLayer = tf.layers.dense({ - inputShape: 64 + this.nbActions, // Previous layer + action + //inputShape: [this.config.batchSize, 64 + this.nbActions], // Previous layer + action units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later @@ -152,34 +236,34 @@ class Critic { // Ouput layer this.outputLayer = tf.layers.dense({ - inputShape: 64, units: 1, kernelInitializer: tf.initializers.randomUniform({ minval: 0.003, maxval: 0.003, seed: this.seed}), - activation: 'tanh', + activation: 'linear', useBias: true, biasInitializer: "zeros" }); // Actor prediction - const predict = () => { + this.predict = () => { return tf.tidy(() => { let l1 = this.firstLayer.apply(obs); - l1 = l1.concat(action); + l1 = this.concat.apply([l1, action]); if (this.firstLayerBatchNorm){ l1 = this.firstLayerBatchNorm.apply(l1); } - //l1 = this.relu(l1); + l1 = this.relu1.apply(l1); + let l2 = this.secondLayer.apply(l1); if (this.secondLayerBatchNorm){ l2 = this.secondLayerBatchNorm.apply(l2); } - //l2 = this.relu.apply(l2); + l2 = this.relu2.apply(l2); + return this.outputLayer.apply(l2); }); } - const output = predict(); - this.model = tf.model({inputs: this.obs, outputs: output}); + const output = this.predict(); + this.model = tf.model({inputs: [obs, action], outputs: output}); } - }; diff --git a/demo/webapp/public/js/DDPG/noise.js b/demo/webapp/public/js/DDPG/noise.js index c173625..c3a138f 100644 --- a/demo/webapp/public/js/DDPG/noise.js +++ b/demo/webapp/public/js/DDPG/noise.js @@ -6,7 +6,7 @@ * See "C Adapative Scaling" Page 14 in the paper. */ -class AdaptiveParamNoiseSpec{ +class AdaptiveParamNoiseSpec { /** * @param conf Object @@ -19,7 +19,7 @@ class AdaptiveParamNoiseSpec{ this.initialStddev = conf.initialStddev || 0.1; this.desiredActionStddev = conf.initialStddev || 0.1; this.adoptionCoefficient = conf.adoptionCoefficient || 1.01; - this.currentStddev = conf.initialStddev; + this.currentStddev = this.initialStddev; } /** diff --git a/demo/webapp/public/js/viewer.js b/demo/webapp/public/js/viewer.js index dba4376..6994e0a 100644 --- a/demo/webapp/public/js/viewer.js +++ b/demo/webapp/public/js/viewer.js @@ -112,4 +112,51 @@ function displayState(id, state, width, height){ yPos += ySize; } +} + +METRICS = {}; + +function initMetricsContainer(container, metrics){ + container = document.getElementById(container); + for (let m=0; m < metrics.length; m++){ + nDiv = document.createElement("div"); + nDiv.id = 'metrics_'+metrics[m]; + nDiv.style.width = "250px"; + nDiv.style.height = "250px"; + nDiv.style.display = "inline-block"; + nDiv.style.marginRight = "10px"; + container.appendChild(nDiv); + + METRICS[metrics[m]] = new CanvasJS.Chart('metrics_'+metrics[m], { + width: 250, + height: 250, + animationEnabled: false, + theme: "light2", + title:{ + text: metrics[m], + }, + axisY:{ + includeZero: false + }, + data: [{ + type: "line", + dataPoints: [{y: 0}] + }] + }); + METRICS[metrics[m]].render(); + } +} + +function setMetric(name, value){ + let chart = METRICS[name]; + let size = chart.options.data[0].dataPoints.length - 1; + + if (chart.options.data[0].dataPoints.length > 500){ + chart.options.data[0].dataPoints = chart.options.data[0].dataPoints.slice(1, size); + size = size - 1; + } + + size = chart.options.data[0].dataPoints.length - 1; + chart.options.data[0].dataPoints.push({y: value, x: chart.options.data[0].dataPoints[size].x+1}); + chart.render(); } \ No newline at end of file diff --git a/src/car.ts b/src/car.ts index 23aae93..32be0e6 100644 --- a/src/car.ts +++ b/src/car.ts @@ -11,6 +11,7 @@ import { CAR_IMG, Sprite, MAP, ROADSIZE, Container, Graphics } from "./global"; import { RoadSprite } from "./asset_manager"; +import { runInThisContext } from "vm"; var Global_carId = 0; @@ -55,6 +56,10 @@ export interface CarSprite extends PIXI.Sprite { optionalTurn?: boolean; agent?: boolean; v?: number; + a?: number; + yaw_rate?: number; + last_a?: number; + last_yaw_rate?: number; } export interface LidarChild extends PIXI.Graphics { @@ -62,6 +67,14 @@ export interface LidarChild extends PIXI.Graphics { pt?: boolean; } +export interface State { + lidar?: number[][]; + linear?: number[]; + v?: number; + a?: number; + steering?: number; +} + export class Car { public level: Level|Editor; @@ -173,6 +186,8 @@ export class Car { Method used to restore the position of the car to the original position (as set into the json file) */ + this.core.a = 0; + this.core.v = 0; this.core.mx = this.info.mx; this.core.my = this.info.my; let road = this.level.getRoad(this.core.my, this.core.mx); @@ -181,19 +196,26 @@ export class Car { } } - getState(linear:boolean = false): number[][]|number[]{ + getState(): State { /* Get the current state of the car The state is the current value of each point of the lidar. */ - if (!linear) - return this.motion.state.map(function(arr: any) { return arr.slice(); }); - else{ - let state: number[] = []; - this.motion.state.map((row: number[]) => { state = state.concat(row);}); - return state; - } + let nState: State = {}; + + let linear: number[] = []; + this.motion.state.map((row: number[]) => { linear = linear.concat(row);}); + linear.push(this.core.v); + + nState.linear = linear; + nState.lidar = this.motion.state.map(function(arr: any) { return arr.slice(); }); + + nState.v = this.core.v; + nState.a = this.core.last_a; + nState.steering = this.core.last_yaw_rate; + + return nState; } step(delta: number, action:number|number[]=null){ diff --git a/src/control_motion_engine.ts b/src/control_motion_engine.ts index 73f93cc..bb68b3b 100644 --- a/src/control_motion_engine.ts +++ b/src/control_motion_engine.ts @@ -10,6 +10,7 @@ import * as U from "./utils"; export class ControlMotionEngine extends MotionEngine { private actions: (string|number)[]; + private maxSpeed: number = 1.5; constructor(level: Level) { /* @@ -133,6 +134,9 @@ export class ControlMotionEngine extends MotionEngine { Step into the environement @delta (Float) time since the last update */ + this.car.last_a = this.car.a; + this.car.last_yaw_rate = this.car.yaw_rate; + // The car lose speed over time if (this.car.v > 0 && this.car.a == 0) this.car.v = Math.max(0, this.car.v - 0.01); @@ -140,14 +144,14 @@ export class ControlMotionEngine extends MotionEngine { this.car.v = Math.min(0, this.car.v + 0.01); if (this.car.a > 0 && this.car.v >= 0) - this.car.v = Math.min(1.5, this.car.v + this.car.a*0.01); + this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.01); else if (this.car.a > 0 && this.car.v < 0){ - this.car.v = Math.min(1.5, this.car.v + this.car.a*0.03); + this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.03); } if (this.car.a < 0 && this.car.v <= 0) - this.car.v = Math.max(-1.5, this.car.v + this.car.a*0.01); + this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.01); else if (this.car.a < 0 && this.car.v > 0){ - this.car.v = Math.max(-1.5, this.car.v + this.car.a*0.03); + this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.03); } if (this.car.yaw_rate == 0){ this.car.x += this.car.v * Math.cos(this.car.rotation)*delta; diff --git a/src/level.ts b/src/level.ts index d675f92..a181c8c 100644 --- a/src/level.ts +++ b/src/level.ts @@ -110,12 +110,7 @@ export class Level extends World { } setReward(agent_col: any, on_road: any, action: any){ - /* - TODO: Let's the reward define in the agent class - */ - let reward = -0.1; - if (action == 0 || this.agent.core.v == 1) - reward += 0.5; + let reward = -0.8 + this.agent.core.v / this.agent.motion.maxSpeed; if (agent_col.length > 0){ reward = -1; } @@ -133,12 +128,16 @@ export class Level extends World { } - step(delta: number, action:number|number[]=null){ + step(delta: number, action:number|number[]=null, auto: boolean = true){ /* Process one step into the environement @delta (Float) time since the last update @action: (Integer) The action to take (can be null if no action) */ + if (auto && !this.steping){ + return; + } + // Go through all cars to move each one for (var c = 0; c < this.cars.length; c++) { if (this.cars[c].lidar && !this.cars[c].core.agent) // If this car can move diff --git a/src/metacar.ts b/src/metacar.ts index 9628ff2..e3a32bc 100644 --- a/src/metacar.ts +++ b/src/metacar.ts @@ -8,7 +8,7 @@ import {UIEvent} from "./ui_event"; import * as U from "./utils"; import { BasicMotionEngine, BasicMotionOptions } from "./basic_motion_engine"; import { ControlMotionEngine } from "./control_motion_engine"; -import { LidarInfoI } from "./car"; +import { LidarInfoI, State } from "./car"; /** * @local Chooce whether to load a file from the computer. @@ -132,6 +132,14 @@ export class MetaCar { this.level.render(val); } + /** + * Choose wheter the environment should step automaticly + * @param val True or False + */ + public steping(val: boolean){ + this.level.setSteping(val); + } + /** * Usefull method to save/download a string as file. * @content The content of the file @@ -157,8 +165,8 @@ export class MetaCar { * The size of the state depends of the size of the Lidar. * @return The state as a 2D Array or 1D Array (linear:true) */ - public getState(linear:boolean = false): number[][]|number[]{ - return this.level.agent.getState(linear); + public getState(): State{ + return this.level.agent.getState(); } /** @@ -167,7 +175,7 @@ export class MetaCar { @return Reward value */ public step(action: number|number[]): number{ - return this.level.step(1, action); + return this.level.step(1, action, false); } /** diff --git a/src/ui_event.ts b/src/ui_event.ts index 929db62..0b2b6cd 100644 --- a/src/ui_event.ts +++ b/src/ui_event.ts @@ -61,6 +61,7 @@ export class UIEvent { // Listen the event button.addEventListener("click", () => { this.level.render(false); + this.level.setSteping(false); if (fc) fc(); }); } diff --git a/src/world.ts b/src/world.ts index ac9481e..0d76078 100644 --- a/src/world.ts +++ b/src/world.ts @@ -21,6 +21,7 @@ export class World { protected loop: any; // Loop method called for each render protected canvasId: string; // Id of the target canvas protected cars: Car[] = []; + protected steping: boolean = true; constructor(levelContent: LevelInfo, canvasId: string) { /* @@ -124,9 +125,11 @@ export class World { render(val: boolean){ if (val){ this.app.ticker.start(); + this.steping = true; } else{ this.app.ticker.stop(); + this.steping = false; } } @@ -144,4 +147,10 @@ export class World { return this.app.renderer.plugins.interaction.mouse.global; } + /** + * Stop stepping in the environment automaticly + */ + public setSteping(val: boolean): void{ + this.steping = val; + } } \ No newline at end of file From 9342208dc77ec98f0bc607fe98764cab4efa7bf4 Mon Sep 17 00:00:00 2001 From: Thibault Neveu Date: Mon, 25 Jun 2018 09:38:11 +0100 Subject: [PATCH 3/9] Steering Angle: works --- demo/webapp/public/js/DDPG/ddpg.js | 188 +++++++---- demo/webapp/public/js/DDPG/ddpg_agent.js | 110 ++++--- demo/webapp/public/js/DDPG/index.js | 43 ++- demo/webapp/public/js/DDPG/models.js | 133 +++----- demo/webapp/public/js/DDPG/noise.js | 4 +- demo/webapp/public/js/editor.js | 2 +- src/asset_manager.ts | 7 +- src/car.ts | 2 - src/control_motion_engine.ts | 6 +- src/embedded/level/level_2.ts | 383 +++++++++++++++++------ src/level.ts | 3 +- 11 files changed, 608 insertions(+), 273 deletions(-) diff --git a/demo/webapp/public/js/DDPG/ddpg.js b/demo/webapp/public/js/DDPG/ddpg.js index 04e95f9..3655528 100644 --- a/demo/webapp/public/js/DDPG/ddpg.js +++ b/demo/webapp/public/js/DDPG/ddpg.js @@ -17,11 +17,10 @@ class DDPG { * @param memory (Memory class) * @param noise (Noise class) */ - constructor(actor, critic, memoryPos, memoryNeg, noise, config){ + constructor(actor, critic, memory, noise, config){ this.actor = actor; this.critic = critic; - this.memoryPos = memoryPos; - this.memoryNeg = memoryNeg; + this.memory = memory; this.noise = noise; this.config = config; this.tfGamma = tf.scalar(config.gamma); @@ -42,6 +41,7 @@ class DDPG { // Randomly Initialize critic network Q(s, a) this.critic.buildModel(obsInput, actionInput); + // Define in js/DDPG/models.js // Init target network Q' and μ' with the same weights this.actorTarget = copyModel(this.actor, Actor); @@ -50,35 +50,38 @@ class DDPG { this.perturbedActor = copyModel(this.actor, Actor); //this.adaptivePerturbedActor = copyModel(this.actor, Actor); + this.criticWithActor = (tfState) => { + return tf.tidy(() => { + const tfAct = this.actor.predict(tfState); + return this.critic.predict(tfState, tfAct); + }); + }; + + this.criticTargetWithActorTarget = (tfState) => { + return tf.tidy(() => { + const tfAct = this.actorTarget.predict(tfState); + return this.criticTarget.predict(tfState, tfAct); + }); + }; + + this.actorOptimiser = tf.train.adam(this.config.actorLr); this.criticOptimiser = tf.train.adam(this.config.criticLr); this.criticWeights = []; - for (let w = 0; w < this.critic.model.weights.length; w++){ - this.criticWeights.push(this.critic.model.weights[w].val); + for (let w = 0; w < this.critic.model.trainableWeights.length; w++){ + this.criticWeights.push(this.critic.model.trainableWeights[w].val); } this.actorWeights = []; - for (let w = 0; w < this.actor.model.weights.length; w++){ - this.actorWeights.push(this.actor.model.weights[w].val); + for (let w = 0; w < this.actor.model.trainableWeights.length; w++){ + this.actorWeights.push(this.actor.model.trainableWeights[w].val); } - // Return a batch from positive and negative experiences - this.memory = { - getBatch: (size) => { - let batch1 = this.memoryPos.getBatch(size/2); - let batch2 = this.memoryNeg.getBatch(size/2); - return { - 'obs0': batch1.obs0.concat(batch2.obs0), - 'obs1': batch1.obs1.concat(batch2.obs1), - 'rewards': batch1.rewards.concat(batch2.rewards), - 'actions': batch1.actions.concat(batch2.actions), - 'terminals': batch1.terminals.concat(batch2.terminals), - }; - } - }; + assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed); - assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev); + this.trainActorCt = 0; + this.trainCriticCt = 0; } @@ -102,17 +105,22 @@ class DDPG { */ adaptParamNoise(){ const batch = this.memory.getBatch(this.config.batchSize); - const tfObs0 = tf.tensor2d(batch.obs0); - const distance = this.distanceMeasure(tfObs0); + let distanceV = null; - assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev); + if (batch.obs0.length > 0){ + const tfObs0 = tf.tensor2d(batch.obs0); + const distance = this.distanceMeasure(tfObs0); + + assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed); + + distanceV = distance.buffer().values; + this.noise.adapt(distanceV[0]); - const distanceV = distance.buffer().values; - this.noise.adapt(distanceV[0]); - setMetric("Distance", distanceV[0]) + distance.dispose(); + tfObs0.dispose(); + } - distance.dispose(); - tfObs0.dispose(); + return distanceV; } /** @@ -176,32 +184,59 @@ class DDPG { const criticLoss = this.criticOptimiser.minimize(() => { const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]); - - const tfAPredictions = this.actorTarget.model.predict(tfObs1); - const tfQPredictions = this.criticTarget.model.predict([tfObs1, tfAPredictions]); - - const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions)); + const tfQPredictions1 = this.criticTargetWithActorTarget(tfObs1); + + const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions1)); - const loss = tf.sub(tfQTargets, tfQPredictions0).square().mean(); - return loss; + return tf.sub(tfQTargets, tfQPredictions0).square().mean(); }, true, this.criticWeights); - setMetric("CriticLoss", criticLoss.buffer().values[0]); + const loss = criticLoss.buffer().values[0]; criticLoss.dispose(); + + //if (this.trainCriticCt % 200 == 0 && this.trainCriticCt != 0){ + targetUpdate(this.criticTarget, this.critic, this.config); + //console.log("Update"); + // Saniity Check + //} + this.trainCriticCt += 1; + + return loss; } - trainActor(tfObs0, it){ - for (let i = 0; i < it; i++){ - const actorLoss = this.actorOptimiser.minimize(() => { - const tfAPredictions0 = this.actor.model.predict(tfObs0); - const tfQPredictions0 = this.critic.model.predict([tfObs0, tfAPredictions0]); - const loss = tf.mean(tfQPredictions0).mul(tf.scalar(-1)); - return loss; - }, true, this.actorWeights); - const vLoss = actorLoss.buffer().values[0]; - setMetric("ActorLoss", actorLoss.buffer().values[0]); - actorLoss.dispose(); + trainActor(tfObs0){ + + const actorLoss = this.actorOptimiser.minimize(() => { + const tfQPredictions0 = this.criticWithActor(tfObs0); + return tf.mean(tfQPredictions0).mul(tf.scalar(-1.)) + }, true, this.actorWeights); + + /* + const sanityTfLoss = tf.tidy(() => { + const tfQPredictions0 = this.criticTargetWithActor(tfObs0); + const mn = tf.mean(tfQPredictions0); + const loss = mn.mul(tf.scalar(-1)); + return loss; + }); + + const sanityLoss = sanityTfLoss.buffer().values[0]; + + + if (sanityLoss == loss){ + console.warn("Sanity check failed. The optimisation have no effet here."); } + */ + + //if (this.trainActorCt % 200 == 0 && this.trainActorCt != 0){ + targetUpdate(this.actorTarget, this.actor, this.config); + //} + //this.trainActorCt += 1; + + const loss = actorLoss.buffer().values[0]; + actorLoss.dispose(); + //sanityTfLoss.dispose(); + + return loss; } getTfBatch(){ @@ -225,27 +260,74 @@ class DDPG { } } - async optimizeCritic(){ + optimizeCritic(){ const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); - this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); + const loss = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); tfActions.dispose(); tfObs0.dispose(); tfObs1.dispose(); tfRewards.dispose(); tfTerminals.dispose(); + + return loss; } - async optimizeActor(it=1){ + optimizeActor(it=1){ const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); - this.trainActor(tfObs0, it); + const loss = this.trainActor(tfObs0, it); tfActions.dispose(); tfObs0.dispose(); tfObs1.dispose(); tfRewards.dispose(); tfTerminals.dispose(); + + return loss; } + + optimizeCriticActor(){ + const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + + const lossC = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); + const lossA = this.trainActor(tfObs0); + + tfActions.dispose(); + tfObs0.dispose(); + tfObs1.dispose(); + tfRewards.dispose(); + tfTerminals.dispose(); + + return {lossC, lossA}; + } + + trainRecord(){ + + let lossValues = []; + + for (let i=0; i < 32; i++){ + + const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + + const actorLoss = this.actorOptimiser.minimize(() => { + const tfAPredictions0 = this.actor.model.predict(tfObs0); + const loss = tfActions.sub(tfAPredictions0).square().mean(); + return loss; + }, true, this.actorWeights); + + const loss = actorLoss.buffer().values[0]; + lossValues.push(loss); + + actorLoss.dispose(); + tfActions.dispose(); + tfObs0.dispose(); + tfObs1.dispose(); + tfRewards.dispose(); + tfTerminals.dispose(); + } + console.log("Mean loss", mean(lossValues)); + } + } \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/ddpg_agent.js b/demo/webapp/public/js/DDPG/ddpg_agent.js index add5eaf..c6a3e51 100644 --- a/demo/webapp/public/js/DDPG/ddpg_agent.js +++ b/demo/webapp/public/js/DDPG/ddpg_agent.js @@ -10,21 +10,23 @@ class DDPGAgent { this.env = env; // Default Config this.config = { - "stateSize": 26, - "nbActions": 2, - "layerNom": true, + "stateSize": 17, + "nbActions": 1, + "layerNorm": false, "normalizeObservations": true, "seed": 0, "criticL2Reg": 0.01, - "batchSize": 32, + "batchSize": 64, "actorLr": 0.0001, "criticLr": 0.001, + "memorySize": 15000, "gamma": 0.99, + "noiseDecay": 0.95, "rewardScale": 1, "nbEpochs": 500, - "nbEpochsCycle": 100, + "nbEpochsCycle": 20, "nbTrainSteps": 50, - "tau": 0.001, + "tau": 0.01, "paramNoiseAdaptionInterval": 50, }; // From js/DDPG/noise.js @@ -34,8 +36,7 @@ class DDPGAgent { // Buffer replay // The baseline use 1e6 but this size should be enough for this problem - this.memoryPos = new Memory(5000); - this.memoryNeg = new Memory(5000); + this.memory = new Memory(this.config.memorySize); // Actor and Critic are from js/DDPG/models.js this.actor = new Actor(this.config); this.critic = new Critic(this.config); @@ -44,9 +45,10 @@ class DDPGAgent { Math.seedrandom(0); this.rewardsList = []; + this.epiDuration = []; // DDPG - this.ddpg = new DDPG(this.actor, this.critic, this.memoryPos, this.memoryNeg, this.noise, this.config); + this.ddpg = new DDPG(this.actor, this.critic, this.memory, this.noise, this.config); } /** @@ -54,11 +56,11 @@ class DDPGAgent { */ play(){ // Get the current state - const state = agent.env.getState().linear; + const state = this.env.getState().linear; // Pick an action - const tfActions = agent.ddpg.predict(tf.tensor2d([state])); + const tfActions = this.ddpg.predict(tf.tensor2d([state])); const actions = tfActions.buffer().values; - agent.env.step([actions[0], actions[1]]); + agent.env.step([1., actions[0]]); tfActions.dispose(); } @@ -85,48 +87,66 @@ class DDPGAgent { stepTrain(tfPreviousStep, mPreviousStep){ // Get actions const tfActions = this.ddpg.perturbedPrediction(tfPreviousStep); + //const TruetfActions = this.ddpg.perturbedPrediction(tfPreviousStep); + //const rdNormal = tf.randomNormal(TruetfActions.shape, 0, this.noisyActions, "float32", this.config.seed); + //const noisyActions = TruetfActions.add(rdNormal); + //const tfActions = tf.clipByValue(noisyActions, -1, 1); + // Step in the environment with theses actions let mAcions = tfActions.buffer().values; - let mReward = this.env.step([mAcions[0], mAcions[1]]); + let mReward = this.env.step([1., mAcions[0]]); this.rewardsList.push(mReward); // Get the new observations let mState = this.env.getState().linear; let tfState = tf.tensor2d([mState]); let mDone = 0; + if (mReward == -1){ mDone = 1; } // Add the new tuple to the buffer - if (mReward >= 0) - this.ddpg.memoryPos.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); - else - this.ddpg.memoryNeg.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); + this.ddpg.memory.append(mPreviousStep, [mAcions[0]], mReward, mState, mDone); // Dispose tensor tfPreviousStep.dispose(); + //TruetfActions.dispose(); + //noisyActions.dispose(); tfActions.dispose(); + //rdNormal.dispose(); + //rd.dispose(); + //trueTfActions.dispose(); return {mDone, mState, tfState} } + initTrainParam(){ + this.stopTraining = false; + this.noisyActions = 2.0; + } + /** * Train DDPG Agent */ async train(realTime){ - this.stopTraining = false; + this.initTrainParam(); // One epoch for (let e=0; e < this.config.nbEpochs; e++){ // Perform cycles. + this.rewardsList = []; + this.stepList = []; + this.distanceList = []; for (let c=0; c < this.config.nbEpochsCycle; c++){ + if (c%10==0){ logTfMemory(); } - this.rewardsList = []; + // Perform rollouts. // Get current observation let mPreviousStep = this.env.getState().linear; let tfPreviousStep = tf.tensor2d([mPreviousStep]); let step = 0; + console.time("LoopTime"); for (step=0; step < 800; step++){ let rel = this.stepTrain(tfPreviousStep, mPreviousStep); @@ -139,30 +159,48 @@ class DDPGAgent { this.env.render(true); return; } - if (realTime) + if (realTime && step % 10 == 0) await tf.nextFrame(); } + this.stepList.push(step); console.timeEnd("LoopTime"); - this.env.reset(); + let distance = this.ddpg.adaptParamNoise(); + this.distanceList.push(distance[0]); + + this.env.randomRoadPosition(); tfPreviousStep.dispose(); - // Mean is define is js/utils.js console.log("e="+ e +", c="+c); - setMetric("Reward", mean(this.rewardsList)); - setMetric("EpisodeDuration", step); - this.ddpg.adaptParamNoise(); + + //this.ddpg.targetUpdate(); await tf.nextFrame(); } - console.time("LoopTrain"); - for (let t=0; t < 100; t++){ - this.ddpg.optimizeCritic(); - } - for (let t=0; t < 100; t++){ - this.ddpg.optimizeActor(); - } - console.timeEnd("LoopTrain"); - this.ddpg.targetUpdate(); + if (this.ddpg.memory.length == this.config.memorySize){ + this.noisyActions = Math.max(0.1, this.noisyActions * this.config.noiseDecay); + this.ddpg.noise.desiredActionStddev = Math.min(0.5, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev); + let lossValuesCritic = []; + let lossValuesActor = []; + console.time("Training"); + for (let t=0; t < 100; t++){ + let lossC = this.ddpg.optimizeCritic(); + lossValuesCritic.push(lossC); + } + for (let t=0; t < 100; t++){ + let lossA = this.ddpg.optimizeActor(); + lossValuesActor.push(lossA); + } + console.timeEnd("Training"); + console.log("desiredActionStddev:", this.ddpg.noise.desiredActionStddev); + setMetric("CriticLoss", mean(lossValuesCritic)); + setMetric("ActorLoss", mean(lossValuesActor)); + } + setMetric("Reward", mean(this.rewardsList)); + setMetric("EpisodeDuration", mean(this.stepList)); + setMetric("Distance", mean(this.distanceList)); + await tf.nextFrame(); + } + + + this.env.render(true); } - this.env.render(true); - } }; \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/index.js b/demo/webapp/public/js/DDPG/index.js index 5ceb838..2810b85 100644 --- a/demo/webapp/public/js/DDPG/index.js +++ b/demo/webapp/public/js/DDPG/index.js @@ -3,8 +3,11 @@ let levelUrl = metacar.level.level2; var env = new metacar.env("canvas", levelUrl); env.setAgentMotion(metacar.motion.ControlMotion, {}); +env.setAgentLidar({pts: 4, width: 2, height: 9, pos: 1}) +RECORD = false; + // js/DDPG/ddpg.js var agent = new DDPGAgent(env); @@ -12,13 +15,21 @@ initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "Epi env.loop(() => { let state = env.getState(); + + if (RECORD){ + agent.ddpg.memory.append(state.linear, [state.a, state.steering], 1, [1, 1, 1], 1); + } + displayState("realtime_viewer", state.lidar, 200, 200); let reward = env.getLastReward(); - const qValue = agent.getQvalue(state.linear, [state.a, state.steering]); + const qValue = agent.getQvalue(state.linear, [state.steering]); displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]); }); + env.load().then(() => { + + // Train agent env.addEvent("train", () => { agent.train(false); @@ -28,6 +39,22 @@ env.load().then(() => { agent.play(); }); + env.addEvent("record", () => { + RECORD = true; + }); + + env.addEvent("randomPos", () => { + env.randomRoadPosition(); + }); + + env.addEvent("stopRecord", () => { + RECORD = false; + }); + + env.addEvent("trainOnRecord", () => { + agent.ddpg.trainRecord(); + }); + env.addEvent("TrainRealTime", () => { env.steping(false); agent.train(true); @@ -38,4 +65,18 @@ env.load().then(() => { }); env.addEvent("reset_env"); + + env.addEvent("load", (content) => { + content = JSON.parse(content); + + for (let c=0; c < content.obs0.length; c++){ + if (content.obs0[c] != 0){ + agent.ddpg.memory.append(content.obs0[c], content.actions[c], content.rewards[c], content.obs0[c], content.terminals[c]); + } + } + + console.log("dataReady"); + + }, {local: true}); }); + diff --git a/demo/webapp/public/js/DDPG/models.js b/demo/webapp/public/js/DDPG/models.js index b47a55d..65c0edb 100644 --- a/demo/webapp/public/js/DDPG/models.js +++ b/demo/webapp/public/js/DDPG/models.js @@ -24,14 +24,14 @@ function copyModel(model, instance){ * @param stddev (number) * @return Copy of the model */ -function assignAndStd(actor, perturbedActor, stddev){ +function assignAndStd(actor, perturbedActor, stddev, seed){ return tf.tidy(() => { - const weights = actor.model.weights; + const weights = actor.model.trainableWeights; for (let m=0; m < weights.length; m++){ - let shape = perturbedActor.model.weights[m].val.shape; - let randomTensor = tf.randomNormal(shape, 0, stddev); + let shape = perturbedActor.model.trainableWeights[m].val.shape; + let randomTensor = tf.randomNormal(shape, 0, stddev, "float32", seed); let nValue = weights[m].val.add(randomTensor); - perturbedActor.model.weights[m].val.assign(nValue); + perturbedActor.model.trainableWeights[m].val.assign(nValue); } }); } @@ -44,10 +44,10 @@ function assignAndStd(actor, perturbedActor, stddev){ */ function assignModel(model, targetModel){ return tf.tidy(() => { - const weights = model.model.weights; + const weights = model.model.trainableWeights; for (let m=0; m < weights.length; m++){ let nValue = weights[m].val; - targetModel.model.weights[m].val.assign(nValue); + targetModel.model.trainableWeights[m].val.assign(nValue); } }); } @@ -62,15 +62,20 @@ function assignModel(model, targetModel){ */ function targetUpdate(target, original, config){ return tf.tidy(() => { - const originalW = original.model.weights; - const targetW = target.model.weights; + const originalW = original.model.trainableWeights; + const targetW = target.model.trainableWeights; const one = tf.scalar(1); const tau = tf.scalar(config.tau); for (let m=0; m < originalW.length; m++){ + const lastValue = target.model.trainableWeights[m].val.clone(); let nValue = tau.mul(originalW[m].val).add(targetW[m].val.mul(one.sub(tau))); - target.model.weights[m].val.assign(nValue); + target.model.trainableWeights[m].val.assign(nValue); + const diff = lastValue.sub(target.model.trainableWeights[m].val).mean().buffer().values; + if (diff[0] == 0){ + console.warn("targetUpdate: Nothing have been changed!") + } } }); } @@ -97,45 +102,23 @@ class Actor{ buildModel(obs){ this.obs = obs; - - this.firstLayerBatchNorm = null; - this.secondLayerBatchNorm = null; - - this.relu1 = tf.layers.activation({activation: 'relu'}); - //this.relu2 = tf.layers.activation({activation: 'relu'}); - // First layer with BatchNormalization this.firstLayer = tf.layers.dense({ units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), - activation: 'linear', // relu is add later + activation: 'relu', // relu is add later useBias: true, biasInitializer: "zeros" }); - if (this.layerNorm){ - // WARNING: BatchNormalization instead of layerNormalization - this.firstLayerBatchNorm = tf.layers.batchNormalization({ - scale: true, - center: true - }); - } - - /* + // Second layer with BatchNormalization this.secondLayer = tf.layers.dense({ - units: 64, + units: 32, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), - activation: 'linear', // relu is add later + activation: 'relu', // relu is add later useBias: true, biasInitializer: "zeros" }); - if (this.layerNorm){ - // WARNING: BatchNormalization instead of layerNormalization - this.secondLayerBatchNorm = tf.layers.batchNormalization({ - scale: true, - center: true - }); - }*/ // Ouput layer this.outputLayer = tf.layers.dense({ @@ -147,22 +130,15 @@ class Actor{ biasInitializer: "zeros" }); - this.predict = () => { - return tf.tidy(() => { - let l1 = this.firstLayer.apply(obs); - if (this.firstLayerBatchNorm){ - l1 = this.firstLayerBatchNorm.apply(l1); + this.predict = (tfState) => { + return tf.tidy(() => { + if (tfState){ + obs = tfState; } - l1 = this.relu1.apply(l1); - /* + let l1 = this.firstLayer.apply(obs); let l2 = this.secondLayer.apply(l1); - if (this.secondLayerBatchNorm){ - l2 = this.secondLayerBatchNorm.apply(l2); - } - l2 = this.relu2.apply(l2); - */ - - return this.outputLayer.apply(l1); + + return this.outputLayer.apply(l2); }); } const output = this.predict(); @@ -194,45 +170,35 @@ class Critic { this.obs = obs; this.action = action; - this.firstLayerBatchNorm = null; - this.secondLayerBatchNorm = null; + this.add = tf.layers.add(); - this.relu1 = tf.layers.activation({activation: 'relu'}); - this.relu2 = tf.layers.activation({activation: 'relu'}); - this.concat = tf.layers.concatenate(); + // First layer with BatchNormalization + this.firstLayerS = tf.layers.dense({ + units: 64, + kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), + activation: 'linear', // relu is add later + useBias: true, + biasInitializer: "zeros" + }); // First layer with BatchNormalization - this.firstLayer = tf.layers.dense({ + this.firstLayerA = tf.layers.dense({ units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later useBias: true, biasInitializer: "zeros" }); - if (this.layerNorm){ - // WARNING: BatchNormalization instead of layerNormalization - this.firstLayerBatchNorm = tf.layers.batchNormalization({ - scale: true, - center: true - }); - } // Second layer with BatchNormalization this.secondLayer = tf.layers.dense({ //inputShape: [this.config.batchSize, 64 + this.nbActions], // Previous layer + action - units: 64, + units: 32, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), - activation: 'linear', // relu is add later + activation: 'relu', useBias: true, biasInitializer: "zeros" }); - if (this.layerNorm){ - // WARNING: BatchNormalization instead of layerNormalization - this.secondLayerBatchNorm = tf.layers.batchNormalization({ - scale: true, - center: true - }); - } // Ouput layer this.outputLayer = tf.layers.dense({ @@ -245,22 +211,21 @@ class Critic { }); // Actor prediction - this.predict = () => { + this.predict = (tfState, tfActions) => { return tf.tidy(() => { - let l1 = this.firstLayer.apply(obs); - l1 = this.concat.apply([l1, action]); - if (this.firstLayerBatchNorm){ - l1 = this.firstLayerBatchNorm.apply(l1); + if (tfState && tfActions){ + obs = tfState; + action = tfActions; } - l1 = this.relu1.apply(l1); + + let l1A = this.firstLayerA.apply(action); + let l1S = this.firstLayerS.apply(obs) - let l2 = this.secondLayer.apply(l1); - if (this.secondLayerBatchNorm){ - l2 = this.secondLayerBatchNorm.apply(l2); - } - l2 = this.relu2.apply(l2); + let concat = this.add.apply([l1A, l1S]) - return this.outputLayer.apply(l2); + let l2 = this.secondLayer.apply(concat); + + return this.outputLayer.apply(l2); }); } const output = this.predict(); diff --git a/demo/webapp/public/js/DDPG/noise.js b/demo/webapp/public/js/DDPG/noise.js index c3a138f..9c0ebb2 100644 --- a/demo/webapp/public/js/DDPG/noise.js +++ b/demo/webapp/public/js/DDPG/noise.js @@ -16,8 +16,8 @@ class AdaptiveParamNoiseSpec { */ constructor(conf){ conf = conf || {}; - this.initialStddev = conf.initialStddev || 0.1; - this.desiredActionStddev = conf.initialStddev || 0.1; + this.initialStddev = conf.initialStddev || 0.3; + this.desiredActionStddev = conf.desiredActionStddev || 0.3; this.adoptionCoefficient = conf.adoptionCoefficient || 1.01; this.currentStddev = this.initialStddev; } diff --git a/demo/webapp/public/js/editor.js b/demo/webapp/public/js/editor.js index 97b0e54..3a569b3 100644 --- a/demo/webapp/public/js/editor.js +++ b/demo/webapp/public/js/editor.js @@ -16,5 +16,5 @@ editor.load().then(() => { localStorage.setItem('mylevel.json', JSON.stringify(content)); window.open("/test_editor.html"); - }, {download: false, name: "level.json"}); + }, {download: true, name: "level.json"}); }); \ No newline at end of file diff --git a/src/asset_manager.ts b/src/asset_manager.ts index 8a3c97c..c93474d 100644 --- a/src/asset_manager.ts +++ b/src/asset_manager.ts @@ -140,6 +140,9 @@ export class AssetManger { } car.line = line; + car.a = 0; + car.v = 0; + // Set the car on the road car.x = road.x; car.y = road.y; @@ -158,7 +161,6 @@ export class AssetManger { let line_side_factor_t = line == 0 ? 0:Math.PI; car.x += line_side_factor*Math.round(x_m); car.y += line_side_factor*Math.round(y_m); - // (x, y position) Relatif to the map car.mx = Math.floor(car.x / ROADSIZE); car.my = Math.floor(car.y / ROADSIZE); @@ -166,6 +168,9 @@ export class AssetManger { car.rotation = (th+line_side_factor_t); // Set the new road of the car car.checkAndsetNewRoad(road); + + + } createAsset(img: string, info: AssetInfo, textures: any, type: string){ diff --git a/src/car.ts b/src/car.ts index 32be0e6..a4a6309 100644 --- a/src/car.ts +++ b/src/car.ts @@ -186,8 +186,6 @@ export class Car { Method used to restore the position of the car to the original position (as set into the json file) */ - this.core.a = 0; - this.core.v = 0; this.core.mx = this.info.mx; this.core.my = this.info.my; let road = this.level.getRoad(this.core.my, this.core.mx); diff --git a/src/control_motion_engine.ts b/src/control_motion_engine.ts index bb68b3b..437269c 100644 --- a/src/control_motion_engine.ts +++ b/src/control_motion_engine.ts @@ -10,7 +10,7 @@ import * as U from "./utils"; export class ControlMotionEngine extends MotionEngine { private actions: (string|number)[]; - private maxSpeed: number = 1.5; + private maxSpeed: number = 2.0; constructor(level: Level) { /* @@ -144,12 +144,12 @@ export class ControlMotionEngine extends MotionEngine { this.car.v = Math.min(0, this.car.v + 0.01); if (this.car.a > 0 && this.car.v >= 0) - this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.01); + this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.005); else if (this.car.a > 0 && this.car.v < 0){ this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.03); } if (this.car.a < 0 && this.car.v <= 0) - this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.01); + this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.005); else if (this.car.a < 0 && this.car.v > 0){ this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.03); } diff --git a/src/embedded/level/level_2.ts b/src/embedded/level/level_2.ts index 49a32f8..88a98ea 100644 --- a/src/embedded/level/level_2.ts +++ b/src/embedded/level/level_2.ts @@ -1,114 +1,319 @@ export const level2: any = { - "cars": [ + "cars": [], + "road": [ { - "mx": 7, - "my": 7, - "line": 0 + "x": 300, + "y": 360 }, { - "mx": 9, - "my": 6, - "line": 0 + "x": 300, + "y": 360 }, { - "mx": 8, - "my": 6, - "line": 1 + "x": 360, + "y": 360 }, { - "mx": 10, - "my": 6, - "line": 1 - } - ], - "house": [ + "x": 420, + "y": 360 + }, + { + "x": 240, + "y": 360 + }, + { + "x": 180, + "y": 360 + }, + { + "x": 120, + "y": 360 + }, + { + "x": 480, + "y": 360 + }, + { + "x": 480, + "y": 360 + }, + { + "x": 120, + "y": 360 + }, + { + "x": 60, + "y": 360 + }, + { + "x": 300, + "y": 300 + }, + { + "x": 300, + "y": 240 + }, + { + "x": 480, + "y": 360 + }, + { + "x": 480, + "y": 360 + }, + { + "x": 540, + "y": 360 + }, + { + "x": 300, + "y": 180 + }, + { + "x": 360, + "y": 180 + }, + { + "x": 240, + "y": 180 + }, + { + "x": 240, + "y": 180 + }, + { + "x": 180, + "y": 180 + }, + { + "x": 420, + "y": 180 + }, + { + "x": 420, + "y": 120 + }, + { + "x": 180, + "y": 120 + }, + { + "x": 300, + "y": 60 + }, + { + "x": 420, + "y": 60 + }, + { + "x": 180, + "y": 60 + }, + { + "x": 240, + "y": 60 + }, + { + "x": 360, + "y": 60 + }, { - "x": 68, - "y": 317 + "x": 480, + "y": 60 + }, + { + "x": 120, + "y": 60 + }, + { + "x": 540, + "y": 60 + }, + { + "x": 540, + "y": 120 + }, + { + "x": 540, + "y": 180 + }, + { + "x": 540, + "y": 240 + }, + { + "x": 540, + "y": 300 + }, + { + "x": 60, + "y": 60 + }, + { + "x": 60, + "y": 120 + }, + { + "x": 60, + "y": 180 + }, + { + "x": 60, + "y": 240 + }, + { + "x": 60, + "y": 300 + }, + { + "x": 60, + "y": 240 + }, + { + "x": 360, + "y": 180 + }, + { + "x": 480, + "y": 360 } ], - "house2": [ + "asset": [ { - "x": 624, - "y": 84 + "x": 162, + "y": 264 }, { - "x": -15, - "y": 144 + "x": 407, + "y": 264 + }, + { + "x": 259, + "y": 126 + }, + { + "x": 306, + "y": 33 + }, + { + "x": 491, + "y": 148 + }, + { + "x": 391, + "y": 431 + }, + { + "x": 608, + "y": 35 + }, + { + "x": 484, + "y": 224 + }, + { + "x": 248, + "y": 297 + }, + { + "x": 376, + "y": 336 + }, + { + "x": 78, + "y": 429 + }, + { + "x": 1, + "y": 205 + }, + { + "x": 3, + "y": 106 + }, + { + "x": -46, + "y": 290 + }, + { + "x": 622, + "y": 154 + }, + { + "x": 617, + "y": 280 } ], - "house3": [ + "house": [ { - "x": 618, - "y": 231 + "x": 162, + "y": 264 }, { - "x": 13, - "y": 441 + "x": 622, + "y": 154 } ], - "bench": [ + "house3": [ { - "x": 318, - "y": 216 + "x": 407, + "y": 264 }, { - "x": 422, - "y": 36 + "x": -46, + "y": 290 } ], "tree": [ { - "x": 151, - "y": 144 + "x": 259, + "y": 126 }, { - "x": 229, - "y": 175 + "x": 491, + "y": 148 }, { - "x": 326, - "y": 132 + "x": 391, + "y": 431 }, { - "x": 413, - "y": 167 + "x": 608, + "y": 35 }, { - "x": 487, - "y": 157 + "x": 484, + "y": 224 }, { - "x": 445, - "y": 123 + "x": 248, + "y": 297 }, { - "x": 511, + "x": 78, "y": 429 }, { - "x": 584, - "y": 427 - }, - { - "x": 80, - "y": 391 + "x": 1, + "y": 205 }, { - "x": 219, - "y": 409 + "x": 3, + "y": 106 }, { - "x": 207, - "y": 326 - }, + "x": 617, + "y": 280 + } + ], + "bench": [ { - "x": 156, - "y": 426 + "x": 306, + "y": 33 }, { - "x": 162, - "y": 13 + "x": 376, + "y": 336 } ], "map": [ @@ -129,11 +334,11 @@ export const level2: any = { 0, "↱", "↔", + "↧", "↔", "↔", "↔", - "↔", - "↔", + "↧", "↔", "↰", 0 @@ -142,11 +347,11 @@ export const level2: any = { 0, "↕", 0, + "↕", 0, 0, 0, - 0, - 0, + "↕", 0, "↕", 0 @@ -154,32 +359,19 @@ export const level2: any = { [ 0, "↕", - 0, - 0, - 0, - 0, - 0, - 0, - 0, - "↕", - 0 - ], - [ 0, "↳", "↔", - "↔", - "↔", "↧", - "↔", - "↔", - "↔", + "↠", "↲", + 0, + "↕", 0 ], [ 0, - 0, + "↟", 0, 0, 0, @@ -187,21 +379,34 @@ export const level2: any = { 0, 0, 0, - 0, + "↕", 0 ], [ + 0, + "↕", 0, 0, 0, + "↕", + 0, 0, 0, "↕", + 0 + ], + [ 0, - "↱", + "↳", + "↔", + "↔", "↔", + "↥", "↔", - "↔" + "↔", + "↠", + "↲", + 0 ], [ 0, @@ -209,17 +414,17 @@ export const level2: any = { 0, 0, 0, - "↕", 0, - "↕", + 0, + 0, 0, 0, 0 ] ], "agent": { - "mx": 2, - "my": 4, + "mx": 4, + "my": 3, "line": 0, "motion": { "type": "BasicMotionEngine", diff --git a/src/level.ts b/src/level.ts index a181c8c..605640c 100644 --- a/src/level.ts +++ b/src/level.ts @@ -110,7 +110,8 @@ export class Level extends World { } setReward(agent_col: any, on_road: any, action: any){ - let reward = -0.8 + this.agent.core.v / this.agent.motion.maxSpeed; + let reward = 0; + //let reward = -0.8 + this.agent.core.v / this.agent.motion.maxSpeed; if (agent_col.length > 0){ reward = -1; } From ea344a6cb74f80372dc3d40512d547787965f9bf Mon Sep 17 00:00:00 2001 From: Thibault Neveu Date: Mon, 25 Jun 2018 11:18:13 +0100 Subject: [PATCH 4/9] Steering angle and throttle together: works --- demo/webapp/public/js/DDPG/ddpg_agent.js | 14 +++++++------- demo/webapp/public/js/DDPG/index.js | 2 +- demo/webapp/public/js/DDPG/noise.js | 4 ++-- src/level.ts | 3 +-- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/demo/webapp/public/js/DDPG/ddpg_agent.js b/demo/webapp/public/js/DDPG/ddpg_agent.js index c6a3e51..553575b 100644 --- a/demo/webapp/public/js/DDPG/ddpg_agent.js +++ b/demo/webapp/public/js/DDPG/ddpg_agent.js @@ -11,7 +11,7 @@ class DDPGAgent { // Default Config this.config = { "stateSize": 17, - "nbActions": 1, + "nbActions": 2, "layerNorm": false, "normalizeObservations": true, "seed": 0, @@ -19,9 +19,9 @@ class DDPGAgent { "batchSize": 64, "actorLr": 0.0001, "criticLr": 0.001, - "memorySize": 15000, + "memorySize": 20000, "gamma": 0.99, - "noiseDecay": 0.95, + "noiseDecay": 0.99, "rewardScale": 1, "nbEpochs": 500, "nbEpochsCycle": 20, @@ -60,7 +60,7 @@ class DDPGAgent { // Pick an action const tfActions = this.ddpg.predict(tf.tensor2d([state])); const actions = tfActions.buffer().values; - agent.env.step([1., actions[0]]); + agent.env.step([actions[0], actions[1]]); tfActions.dispose(); } @@ -94,7 +94,7 @@ class DDPGAgent { // Step in the environment with theses actions let mAcions = tfActions.buffer().values; - let mReward = this.env.step([1., mAcions[0]]); + let mReward = this.env.step([mAcions[0], mAcions[1]]); this.rewardsList.push(mReward); // Get the new observations let mState = this.env.getState().linear; @@ -106,7 +106,7 @@ class DDPGAgent { } // Add the new tuple to the buffer - this.ddpg.memory.append(mPreviousStep, [mAcions[0]], mReward, mState, mDone); + this.ddpg.memory.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); // Dispose tensor tfPreviousStep.dispose(); @@ -176,7 +176,7 @@ class DDPGAgent { } if (this.ddpg.memory.length == this.config.memorySize){ this.noisyActions = Math.max(0.1, this.noisyActions * this.config.noiseDecay); - this.ddpg.noise.desiredActionStddev = Math.min(0.5, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev); + this.ddpg.noise.desiredActionStddev = Math.max(0.1, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev); let lossValuesCritic = []; let lossValuesActor = []; console.time("Training"); diff --git a/demo/webapp/public/js/DDPG/index.js b/demo/webapp/public/js/DDPG/index.js index 2810b85..6ceed76 100644 --- a/demo/webapp/public/js/DDPG/index.js +++ b/demo/webapp/public/js/DDPG/index.js @@ -22,7 +22,7 @@ env.loop(() => { displayState("realtime_viewer", state.lidar, 200, 200); let reward = env.getLastReward(); - const qValue = agent.getQvalue(state.linear, [state.steering]); + const qValue = agent.getQvalue(state.linear, [state.a, state.steering]); displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]); }); diff --git a/demo/webapp/public/js/DDPG/noise.js b/demo/webapp/public/js/DDPG/noise.js index 9c0ebb2..08c8978 100644 --- a/demo/webapp/public/js/DDPG/noise.js +++ b/demo/webapp/public/js/DDPG/noise.js @@ -16,8 +16,8 @@ class AdaptiveParamNoiseSpec { */ constructor(conf){ conf = conf || {}; - this.initialStddev = conf.initialStddev || 0.3; - this.desiredActionStddev = conf.desiredActionStddev || 0.3; + this.initialStddev = conf.initialStddev || 0.4; + this.desiredActionStddev = conf.desiredActionStddev || 0.4; this.adoptionCoefficient = conf.adoptionCoefficient || 1.01; this.currentStddev = this.initialStddev; } diff --git a/src/level.ts b/src/level.ts index 605640c..f7f12f2 100644 --- a/src/level.ts +++ b/src/level.ts @@ -110,8 +110,7 @@ export class Level extends World { } setReward(agent_col: any, on_road: any, action: any){ - let reward = 0; - //let reward = -0.8 + this.agent.core.v / this.agent.motion.maxSpeed; + let reward = 0 + Math.max(0., this.agent.core.v) / this.agent.motion.maxSpeed; if (agent_col.length > 0){ reward = -1; } From ff27eae6dd638ec5c59b0a87c4afb363eaf988f2 Mon Sep 17 00:00:00 2001 From: Thibault Neveu Date: Mon, 25 Jun 2018 16:35:45 +0100 Subject: [PATCH 5/9] Model dumps --- demo/webapp/public/js/DDPG/ddpg.js | 19 ---------------- demo/webapp/public/js/DDPG/ddpg_agent.js | 16 +++++++++++++ demo/webapp/public/js/DDPG/index.js | 21 +++++------------- demo/webapp/public/js/level1.js | 2 +- .../models/ddpg/actor-model-ddpg-agent.json | 1 + .../ddpg/actor-model-ddpg-agent.weights.bin | Bin 0 -> 13192 bytes .../models/ddpg/critic-model-ddpg-agent.json | 1 + .../ddpg/critic-model-ddpg-agent.weights.bin | Bin 0 -> 13828 bytes 8 files changed, 25 insertions(+), 35 deletions(-) create mode 100644 demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json create mode 100644 demo/webapp/public/models/ddpg/actor-model-ddpg-agent.weights.bin create mode 100644 demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json create mode 100644 demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin diff --git a/demo/webapp/public/js/DDPG/ddpg.js b/demo/webapp/public/js/DDPG/ddpg.js index 3655528..63fd1cc 100644 --- a/demo/webapp/public/js/DDPG/ddpg.js +++ b/demo/webapp/public/js/DDPG/ddpg.js @@ -211,26 +211,7 @@ class DDPG { return tf.mean(tfQPredictions0).mul(tf.scalar(-1.)) }, true, this.actorWeights); - /* - const sanityTfLoss = tf.tidy(() => { - const tfQPredictions0 = this.criticTargetWithActor(tfObs0); - const mn = tf.mean(tfQPredictions0); - const loss = mn.mul(tf.scalar(-1)); - return loss; - }); - - const sanityLoss = sanityTfLoss.buffer().values[0]; - - - if (sanityLoss == loss){ - console.warn("Sanity check failed. The optimisation have no effet here."); - } - */ - - //if (this.trainActorCt % 200 == 0 && this.trainActorCt != 0){ targetUpdate(this.actorTarget, this.actor, this.config); - //} - //this.trainActorCt += 1; const loss = actorLoss.buffer().values[0]; actorLoss.dispose(); diff --git a/demo/webapp/public/js/DDPG/ddpg_agent.js b/demo/webapp/public/js/DDPG/ddpg_agent.js index 553575b..2a6050c 100644 --- a/demo/webapp/public/js/DDPG/ddpg_agent.js +++ b/demo/webapp/public/js/DDPG/ddpg_agent.js @@ -51,6 +51,22 @@ class DDPGAgent { this.ddpg = new DDPG(this.actor, this.critic, this.memory, this.noise, this.config); } + save(env){ + /* + Save the network + */ + this.ddpg.critic.model.save('downloads://critic-model-ddpg-agent'); + this.ddpg.actor.model.save('downloads://actor-model-ddpg-agent'); + } + + async restore(){ + /* + Restore the weights of the network + */ + this.ddpg.critic.model = await tf.loadModel('http://localhost:3000/public/models/ddpg/critic-model-ddpg-agent.json'); + this.ddpg.actor.model = await tf.loadModel("http://localhost:3000/public/models/ddpg/actor-model-ddpg-agent.json"); + } + /** * Play one step */ diff --git a/demo/webapp/public/js/DDPG/index.js b/demo/webapp/public/js/DDPG/index.js index 6ceed76..2eacb40 100644 --- a/demo/webapp/public/js/DDPG/index.js +++ b/demo/webapp/public/js/DDPG/index.js @@ -51,10 +51,6 @@ env.load().then(() => { RECORD = false; }); - env.addEvent("trainOnRecord", () => { - agent.ddpg.trainRecord(); - }); - env.addEvent("TrainRealTime", () => { env.steping(false); agent.train(true); @@ -66,17 +62,12 @@ env.load().then(() => { env.addEvent("reset_env"); - env.addEvent("load", (content) => { - content = JSON.parse(content); - - for (let c=0; c < content.obs0.length; c++){ - if (content.obs0[c] != 0){ - agent.ddpg.memory.append(content.obs0[c], content.actions[c], content.rewards[c], content.obs0[c], content.terminals[c]); - } - } - - console.log("dataReady"); + env.addEvent("save", () => { + agent.save(); + }); - }, {local: true}); + env.addEvent("load", () => { + agent.restore() + }); }); diff --git a/demo/webapp/public/js/level1.js b/demo/webapp/public/js/level1.js index 513f8de..c40dc16 100644 --- a/demo/webapp/public/js/level1.js +++ b/demo/webapp/public/js/level1.js @@ -30,6 +30,6 @@ env.load().then(() => { env.addEvent("save", () => agent.save()); env.addEvent("load", () => { document.getElementById("metacar_canvas_button_train").style.display = "none"; - agent.restore() + agent.restore(); }); }); diff --git a/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json b/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json new file mode 100644 index 0000000..3cc61fd --- /dev/null +++ b/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,17],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-agent.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[17,64],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[32,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.weights.bin b/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.weights.bin new file mode 100644 index 0000000000000000000000000000000000000000..5ed364cb5311567bdd68f6790294d03a80e62c96 GIT binary patch literal 13192 zcmWNX_e02U5XVcY2uVqUv=Ay=>vPx8AWE`*kJ6@@4opXWjn37N@? zWJi&ek#GON^UHI0@Avy&XX|>I9S*XikIuq4^=BdY$*o3{>^`RcU?(WooPjt~E8Gyb zgR;sDam@}RGDtbWo*5)Tx>OY&pLYa1Eke+~G#PWW@39F>`dC$G4J1smp};UtyzQ^e zO}!FAN~uTq6<;?)pz$A;+w$ZM z(`c%KOJBW7Lq`o;r#n;Qume2)8pZw&egTiR2yR8zJa&ho5`hYCoVlH2QPl+^J{YK$!nb@wrniG4Z@;SzH2J)tYrbSDUL5Hu4Gfh zwor%8WjZgA!DUH~MhV+9O#9(xEVpVRF#|$nGJU3bLKCv9Ny@pPumSMr~)S+Ki* zu995tR!qLFh3+0FY2wmR;Ir>MmF0S3`|RS0}55 zX(+lV0nVQB=WMqvrdck}Sd!HedM~*ZQ^&mJd85OS5gx~0R|nC#%km`hU>2Y1xRQK_ zmtuQ_1#TW{q6&8z*eIg|$M!~Yb`Q$A$Y?p*80hb>ZT`9qW4Phi~GR#%&%lh<8&!-Tt{R7HFwBg zn}~n!jUnMqFFbrG2Ko|KW4~ev*f}J^YwrlWb`*sa`Rt7ICRn?om4YXP&@ zqj%P9PO!{}4aaD*_=wd^qdt=a)84>Ehqe5MX#-$kDv64sk<4JR7TMWv<8S{utiLP$ zE)1U6;vcuGL*e^YHX}Wp^UTYDz|=}!-C7N`Z#uIc?FGn!9IiwbDzC0`nQzYayOFQc=sgTPmI4=LVL0G!bZPVXJ4%_W5W zOYLW47L6ivrz)0q&mNxc2_Wl;3ak&?gbOyNv+38A(5*U=ms2*Q#P6+eNu zd(plFo7vZkds(z_E>|ILj*^BK7!wyE?IU|=syJhkcdxShzBcIFHJ@gV(q+j!he8*1 z$T~FvKMN{iM(r4o;tg?<_BZxaLKj7~vT>PP7QT=hW|xL9@`LZ5G082dXkV?@ct7MD zTdF+=L?WIt8U1`*dZ~xCtrLQ8wr(UNI>=qI+JX1f1nIY&3H^1Sg1$%QVqM5#maMi0 zQ}Z=}c5TDo+bgl*VLE=eaDv?L2k=$4yGWsHH%&@0$Dp+t6sUHA*;!A+o;EKuo>;`| z%zOf&`4(7fUdndY`rz&pfqc)dC|F)M3gcJGlCaM?GVCaWmQsDw9 zB`w5c^2;gzx;)(#%*In~-yy*IGg~q99E&!JWOqvc@>h~;sd85Vg}p!7*mdG6H~Die?QhRz^P9`)q+%))8UKg| z<&;?P-dpVKehvDiS&!R}Hp1va5wfrJBDcxk^xA%}BOJGr9=xsMw`}Qv(hb?5pzX$` zoSRJFy_)EzXf`$KsZjIU7|O9creC?o5tgkC0Q>z>l#!kc)7RW3%_)z7yB^9kR*InU z8&RxTmW#H33-DsA*SicSs%w)@_cu6L(Liy( z^_b1t1(2sV1fqM)K-2C3d!N1rWMhxf(%Bq}oA@(fwixm`KQu58q%-wfa7W29im=Qf zfwwCeM*5M3_XB!Wr$HgB!@*eJ1-HH|W)ElA;$$`tdy>z<-@IjPldC$4g`ME`A9SPp zTV&~CSSJ%{)Ig6XTj^NY12A@(0r_nvj*TRSYO@&Rj zQZTaRB9m9v#DFw;DCiMK)tqSh(l8rabu%GYdsAa^eg;Yiijjr6G7dbS$t)yd!Affh zsr8%UWx+U_oDsl3pQB9%Xu@7wdO*lqMRsPCG}gyd>$l?{aJRY9_=OswE|q6b#PldF zVKc~2)&Qw56Ikp0L|8UC9=!HFhTsoMqzt9_cVrIyDK2ESAuCaAYbLgw5az;KS7TCo zFgUB^BB={ODiKjk`a<8FA(K%p!v(B#wqIl~0Y*Lm$9_?L)&4OK9BIWBAlA2?QE9adp49 zp@jA~_#_dGN6V+t`jUfu+0~a!NoXv5(~zZ&+LI~wYdlR|T}F?@AAXg zg8xM9uy@B1^M`cA9kexE?xo?t@NV=gzgjcD*CrE`=f`ITu1>yYrgHny_N z1Lq9Rp-KCl!!m9)M&YSDSl~-CeITS zNXe@L&AXFOJK-Ts=moO(tml%JKStZRXQ;JLj2)K{OlS14_2`vOLB7trof7^V9bFjt2LO6VMmhliRmILv_aG+RpH;W3oH`3SqK z6G_I#m&h;U6jtO#qn@B7&fRnbm5u9Z!?6~c{%sE0-{GiNcsXV#*|BloWk9tpmsUAt zKv#P{U0SP)+IdrHIm||teeyIf=LlPPbpw5DUk$VFm9js(F4IgKE#}pDl{wq5q}8b= zba=#5&R?HPreT%nbo&-BXdQzJ4KtYNaS>MCVnLqf&3xEaWgO}Wp$AVsaP_(oG}%yz z4qoVl%Ll5-_@xrH&57f-=!H-~kt;vDpqc$nU4{x<7t^QU0+y2BOkwROh{inO^y@dH zi0@5)bDtPW3}nEUstQaD2?h7{GF065xqh~lCc@nawq<1*d|DBMN_-SeEau3wZa?fh zW1&CQ`#dN-T!~RuhgjQdT?)&y=X4ZGSX}IQd{exEWG62~Wl?iXJD`kypSEJja5373 zYk}bIHW*r&M6dmK60dGX-fhh!emxXSO`Z8wCU?OnMIPO(O-Mg)J}YQdVv)1$*u~q^ zDZWkx@6DW@td_J8AO&GZ$Bf6|)DCGstdB4wuuk8rS`K z+?duZ$9Yao;;rrCVc%9Y67A2Ssh3}{zVoxNVy`q#IQxw!6;s;!aRmj=air4=iOjCb zQ>ocmrqq~;Ys!RZ|6W~v%SsXAoJYfK(~-I{w}n@GXW-;0#tz+xLiKfVFwQyx&lc>1 zrbqd3Y@;6yd(Xg7ix$@RQxY`IHnL1VL%Q<2fgGCTC|a(9HU{ZY$nP+CbZ;D~{OM(t zDJm#5V?MJFt%gMhMXB-P?A7KV_ysEC;^UBHSg zGeDJ-VGDITcwAeEGpfxfc$692Y<>+c z77K!HvDT<1c?9Da@$ zaZ<*>du{k6R}BBQmyvF?3V8k}W)MGj6Npv(qLt}m(8SHO_j&=V8QQ~aCw{x<=*U}-#1!V ztR6>~b9&h1%_$_hV;TFpu>(cle`MEghJyU0I7(1ACpG67>`?m2d`v#UD9=&~I4;hl zybnOgxPHFS%!c}6r_rgm$3eN?jl4Eqg1F-EY<0u~*e0q=MZeDw-)cqv4LS7j=V|8j z*@{k;T2jaE8=z^tn_d626|QdgVZQOwa7t?jX`D*t+k`}D%U}VyzHNb~o!)rEKopz2 zmeKh12r3!=%W7OQX!vv@Ef35g^^XTBVgC@b9nztTRhCqh+|Tx(G)8HktDNJjDUFpj z=a~HLA23;=4K`G+z|BSZ@cGkrSY2xhK9jmxugWkJ$h5_*bB*A7C>+eyR?%t2G-?|N zXUdgoteKoZDJTi`%FffccPClgG z%epnm(PJ@8fw^cHeubG1NZ{bTWHLB0mK|LCf)B|Rf`U@w6mE>+XIpZx+E;`^(wt%3 z;s=oJphCe8H{kY}(QtYCB>uUcEbr_5y-{ns9Gmpk40VDx&{ywj+|w%#o^|=~$LS8Z zC?24hG4*)tM-%?LX-X*P zYp{0JYOJcxgKdvRpz#k+Vf$Q|fR-0i_qs>!DyOkyA%(@+8;8GpdSO_)SorNPcagcjep$L`!&kHwh- zY#{q7SPddcOJ8BRzo&uE&IQOX`iqiEXWfdS5WP=4zkc8nqr)AY6{W)l+c$d|`U< zzMo$fY=t%MWuQ2i203ls`zMBInxcX=eB`y(s;C))&=x<0xp)S zg^%f5a3F0xsLt<)NhK%X3Y&>>m*!*Dcq9CE`6&iZ@db&Wqsb4M}WoO*C1Z8ise%icwrKkxMknXmtPNh_QMz$>h#c zwq!{-T}a$Q1$K>a#qb8cDvn|U8l{~2b6M#F!AD32;IrDv@O4fxN(%enyJ>s5r(PXwYEd*=T#?4g z+jp3qf(OoQTS`tfzaYm}5;WO~7>Lk!;K8TL5L}+eB0rtp- zLZoa2YvGfan_e59>x-@Xv@ zehl4Q7*2!9nJj&U1m*QSXR6063V!FA zSNz%u<{;9QPbLaw)FM^^IhNH}qMAUOy|K`*c>}g7Swj1(P`oufAD7*^23w`gAbX+E zNSznpROf$CW_=4El%0gK8YhxA4x^J}v&bj*3NL?5fpSukS>SY4dKh{e>|YJBiC<@- zLG@aUaZA9O!ee;!F=}E#3>IClWjK-MknGLpAVK`(77l);&2b_UZ>-kqAD_H zmr-gs0Xm}^ahky-Tr=02H?>%WsJ9%MTQA7?+oFS{7d_J%g(rl4S=5b4CNO<0xfqt< z>}Cg43r*w~E}Rcv&P-1Z@yb2v;eVX!YdwVh`I4=(!9prG7^)XOOk;Yr?zxlI8&mdOf zKG-KO#dCDE&9P5XfHp|3AQ}?jmb@eDQ?-VOHQRLpnu7Wj^&P*lK1@}xS z2dUXcc*83Z*JK;uyPdjRpwk*Goq-s>H5FH7ZebPeH_%|+6WlK?fda6YrG1!!n>Izk zs)fny$Xa9i^V%QIpB{tmiw$AG_BtHYjljr-WoW54k#Whj;9L;|(Hat1_xb?xE#=&^ zZgD(Xv5=(SE`lGyuJj9bQrf(=^h@Ow%iSxFP35!EZAvJ6AG-*5Yu(2GIt{6bZAQzS z7u?0tZhUm)K1`E!L**Mr^tICuw^yfQVADgadw&HQNB8 z4GU9WW8I&5l$z+mX@{3mPj)fRohOKmbH=irLT%`@Aq^h*1i+@OI~cY03Uh2q;W#rN z*!dv~h5`hjFl!ofAE;xOP6|Q}7ms3tO1RwWBQuSBg|$KlS!uKZK2kJhc1fGKrb+{v zHfkH`?2g6IA2;}g;}ZDAORVw1xNRf8VJ>)QCxFyW0bHo6-Pqt�DD1TIT@pSH1KI}I?qiGWhYJU^X6c}^+rdr zOS?uiSy=!kH``#imk!Rox&wt)F6I=jckmBd6uB+4e=@7RCvd6s7ua%b32rmVVP;CF z&{#McJZBj}N}MO`yQhV_P7OkrUn0|7DUL^uT48oc6N=RqQkY*ZIPTLRji$4ZVQ+`8 zf>v<1EUj_!{1mh)umz32M05}-gx9MEKRGj{4{?v zq7bbwO~;6i3Or{KJp&$HpS z-t7hH`dh5e{ux$=1mKK)%dk(+10&yV(GQo|%L)ECjqL)i%=1x?Ugqtoq=XgN!Y#s+ zgB2inax=9Sjl=Z!ub{vu3kSAlH@r^n=hi|LM&I8_JIu21T$TiF8@ z%OToD8Kq_#vKV7wdRC-@{hurN!>+N|ziJoSJ6qC&Kr58n;EpBt{kZ$jZ?b;hFlY}_ zheLMaq+KP7-6OlPl(}PQ!q8-LS=ZYT@KPDGbcFHU{(N@w{!$dU>4pxY6HqTPf}zG# z{1uanr@zXgz%*wZjvw*UF45?~Oq1$6Ea*&yfxn5d0talq0J&hu>Mosz` z6GHWM`uNMs9K%P|@;iU#gVFIHZ0x`3%<=9b9NRIUCJM)!#2@CW`;hnX@+^}CeKU=XF@Ovcc8_SVGu`f%yd6=w6E1|x)E0!*h zrT%MCWG<6Wud^oNIn4^V5?F;ZFQnjq7oC{)9bIM_FqJM`>fpA^eT8tXeCk!7k88Z+ z*~jS;+~1|qTy2UgIi1=?Q3aVa`>GU4OUaR`S34{cOs6HEW$@lEQToD}Hrj=S;7=D( zgOL>u&dJMSt059p#9jEF{!Bb>@E!MR1mnxw@tD207=n|8IWcc9DBErYTR*JE-ebZB z>KPsl@2Uk12IDQs;dT~kx+SnXpCjS87Ak z2YXVO636~7)~0i6D!A~_6Zo!o5z^{_7J7uC{@5W1(3r{I-pfSIBN=$mZ5|rwEg_Z6 zotP$@MlGSE^oL!YVc*VGWQ*C5HenLftrf+H+0#JoLpeQNkj}hSG|+nr;WE2xcyy?T zC8kxQ{p~DX=XyEUIaMA{L@3f0aS0rl^9G|X$y2}JM|9R4jRu`hF*4%_#2=FYo8p(y zJbiQHp^M|8(rp~&RbE3Gvs7y9`-;y7rRnjpW4LdUOr!sCQ!4xZL*Fm>FDI|C3Wu6k zplr%v^t#31p4$z!y2$~Zj1sws7OO^OEg@3&<1qJ43!J{21_v%J;M)sy>D%>{xOsUN z)IW%2@*{b8a;6Oao$rkYR{M@{#zSn6$1-|(Gz3!Wy7;dXLfFmbK%84S4)a!DXuO@- z!TQWh_+7hhK4`7t8IhF34f?8r%vD>#53-3Mz@!hRx{W)C! zgUu5*P1^$}x0Eum4`b+uKrG(9aTc!3u0oxk)@0jah|WqHcqB%LWe*e~)`+2eK@p!C zJ{?2Hn&PE)DKIuJ#lgklcp%{niy1XI!tb*oqf3*VR4h@AJA!ta6LD3+0{m?1fy)fu-Qrv zo11Fb4)IvnzhWI;{#*~@a!tJU!aTOXISF@&Mw6uMBz%2K0xwOx%0!+kf$_ZCeEauV zm}H`hQx8dAZ%(lj=2-3vc+C;r1t3=7A!-i zwR;LKUNj%SiV2|m=~p16cbF51I>7vEbI95*minEN;9utgauDS(`QU0YoXqg;s$z_N z;lk>!&g0$rPt2$1$t=cMKv$>7C$VPIJ8g zh(Arnc)=Z%_V*t1t+r&xwfj)6BpMqGwz7<0mUzgFVd#Z#xb#^IuGIU@NeUHn3pY(? z>qj_L={Y9~I=hpZPW2>gUHA;!)|Y_KiZC1u z6gD{H*2d0m&R~YU(RgS>0ebc;FgvXsyxmeW>>QnqDi!ibM?3L+ks%ru2I1Zl{;*6b z5-0uc#HHG6FuU|0Zx*G9qh`*>S-o;FRdYSQJMkIaSB&h>M{3#RsD z1F!kG36f_Wpukrtq##)hLxK(by2}R0ch%!YiPwD96jPjJa}=(5++f4Saa3M=4__IG zp~xdmED9B)H)mc0m$(d7r>o)Z)25_5)W#Cy=F(zEO$zpn#upYH(B&S1EtjXzTUSTA z#@FNbynIZ&Zw_5baj?Yg4*%-G58mwC7p%Lo29paqn1gOA-fs`5qL1n1(sUgJ#9gSl z&yY@?_Qh-2@7S0AMflBVExD<@#nY{`P%vm5=I;we+YR05+<6^cMNXmSLjzPld6XBC zDZ~?|;p|+-1YC2y0AD)Y(?8oCMZx1n@fur%pdmU4lD^zw@l)K{ZbhJDRafBlyNR$} zGZ;thQ^iApfCERT;bG^CxcvJPJoF6MH`mwvUf)C%Oux!Cq(*V8uc+Y(9Yef$QJQv~ zIzZk*(`oECK|HLggp;gy(!-t`5cRAG4UKy7#i;q@-ua0?HtHI?AFhwh!cqoaOS`#4 zM~1mYmlAQy#$)L2`h<~|G2-D+e5&Yx$vZT#{_hlnFC#tkYN`fvIB(DHjLyZGp?Peb zOBhObq+zbiLUhb9!ZGXCV4J5aE;^}7L*3VKqE0Bvmt|wZ3wf$vYC!V~PQ&h_Gf=tC zkFVW7m+Hf>Gwc2PaN^fYys^3us!JDBe9>1pI{pd_x)-wmNlQpMAw~Pf6MRT1#>}4c zjU_QoEcw6#upV!Wd;djasn`lQ^ep`KF^FdZL$n zA{-tw1G>VEU~SJ;=2X8N9`AU_!YW)EUTWlGkl1J}z4C+OE{@#r%#+6VL$Xwtu$=-X zYG7pPL*C&-8k^y@obRzb%sCW@QB9N_Cbi$-q<*c!=klSnx-@!KsQa5$gMKX|{3>`GH8#8DOw&M!dK z==XJr!4Z&Q8Vw=^cj4Au8@j%HG&Wj1!=H`^@JiuLEM0mNmz?W`n`nDN$)VDsG>Pg)71(@9s1 zzWJ3qRxD<4*YdD`2 zLPrXZalHI3_D({Gb;usackgmoXw5;`5ER0W-wI?8PH2O|@oltrwksa`_7F67x1qDa zZx;ObA75H%PEA`3xgom+Byru87RE|ZoaSlR*y{mL3+5o^2^yFQ#_Oq8$)m#}H=145 zgK1EOo~y=S)D!G+`VfcH7V z?0w7mp{7mL+TO1}Z(29g9iIZWF>6p@RAyKIGFb#=w7U5^Bj;f=g&9=)M~9 zAu(leY&eqM6|LmwjFHE$+z|Gy>W8xrgt1}k33_u>9oEb=qZ=ElIp@@Rgq^7nSDuD? zQ;SeGNt?CG-@~U-c_47A6Zz?>%=7O3kxW0CIUPGhU#y3iipw+%Pz;CJW5lQ_X*JmH zUI({pr{TJ153yj$Mg59f640)d3u69a7?&D~jji!0>t4*SFY09%ZfcQ@!8YVelqm8= zFn;Y5#v1b*s23Iw9>)j3%y<~`+c)Bw7mwj&Ry{l}7~+4=jmFV!!KA7wN-|lgu&u;` z&og&rU&)6F3Pj+oAJ^fBP%<_Z+2S)r0~Bfe0u!7JKp|HM2ktk3N6RI+U@+3l@hfn~ zmln_X{p!eW0gww79_owY;R{3urkd zOc(!-Xdz2PujHMmk}89D&24dCUkuF6wr00pOVR!Av$)CM5p{080WV!W%CFVL0#6^3 z@fOw}meB&a_wSjTvM0&xP5_?sAP;_ooBvG3aH&{)FB5?RQjb94S}0w2XDH&a29nc* zF!aw_G7T!hrR~+sK7Jh;h*^%zc1cR-E7@j?XzI&ZfrBq2nCs75cxH7Aj5@y%`>zJz zB41rR_jHm$TES_2yO|&@cMnX|io}l-hVX#48KoA?FlcV)&@1s2DBx24R3$yM{62-c z7V~IRb`Vr`pP+a5KDP1JJ8b?po2)*61EKPxc)mFGnbl+rq#$g~mNK}sb|uE0 zSPSD^Wejvnq+sp&nFdNX#0=JC7O~YnSJ^D@kga;0UVnK_-NY>JT;QC;!Ja30H zm;OPUWl{7|cVte;ub`$VLs)aJil%lPpf~Ro>GkO?0PWL$pAoO@#N`-z+E zb$c5qtG~pY_09NmgrAmOm!Q{b?wD;d1AH1s8BkmnyiiSmtCxH5mB(mwQ+%vHs<4Lr zyKsy?3>ToAiwo)gS%p6&6G&rZN4C47mrJk;LF3OqK{BWsKPfK90f{|W`XGv)W;LSM z_JibmryR|0{YT;c-YBxhiG_$3k=)J#>J%B!{~&#-QFE0HJpCp|6F1q@`17)8Cb$H1 z`5^9vwIlj@XR-65y_}WyY4W-@9(tta;>Tw>Ecj`kPJwBxxITmiX`NTl*+=K8B0#ZA=khvF^tDI3+;*f|rI)~=(*_o_H;`Eyom|C`yJ zw4gaY#rpPsvZ(1(iQkr=K{@3VO7YPG?UJLcc5n(7mR@I>>t|3zaV&N6{t(%J0d{FC z^7mj6o)w1T&$eq^#N$!$R74gPg< z>W<8ygiyL1AVIr&jOcV#Bi>cdK;4PepkK$Za855+bSKi|#B=!WKnLu6kdJ;dw^7Nt zSzN-FN8k~YM+f+0&{!r7moGkJA55oVh4^CTP#a79A`uGu_5nM%hp5^$5ADYO1NJGH z0@+9QPkRZ=6?@HT=vvS-ld&XMo-}fk9NnOq@fa`rIJ&o#yREQuT?W6^J(LEQ)o_VY@3~L?aiHH4Kv^H3vb9&| zQg)LWJqtbp7cOq*s=^y!`P>`q)7*^|qqPG)@14ObMQXUDPaQw*Ujlh%qp+cV8;URT z;nu-IHsg;K+1Yr&F3T9Y5MaU%+kaq7|7BrFu?{KL?Z<7$@36D2Z^3h_Aq&}~z{*mT zQRQnAdli_AuLMVWz0;fB))!+VUxt6Q{&1h@2-@6M!?CxDnf`Src-MakN3RvaEhAa= zn}HYJ{;p2jZFAWA^km-m(n>mGR|^@R*Wh5R24AtD6b19n(mBV^xO;IN{kwmVZFnRH z9(paP>o0@4UOVts)mro|e9S~$%Q4XEGc$g>8RIG|*>;I$2w5G4;;DBbx$g{&uYJq^ z3r%N_l$O$~>RhPJP{#N*E!=yRJM5z0dUiLk2D(nwqoZOo&YnBOPUR|~gS9rV@v)Ot z23SE`l?}^RTZ}UM=HQnV51Ic;0W|Wn!u3g!Eb2dP%vJTkF*(n0->!@Q;BR`6AXBeW>q)^;CGX`aSRY>4Ehryn+Nn7#=`?Spw zCtKOFaV~eDynH*|ku}7lR$X|XoyB)vQM7LJbC}<1fgM)Q_-W-IQL#T8ntq3}1sB}e z&yG$EBSOZ=#h{P+|Rv7c*F{@moif#?ESo3`*LOmaP=U#=AFQn1c zN7A&>%9vkvTMA#8kL1iIH5F>diB1xHdwUbM zlnlX)^@2D~{vNc&KZ5dsPjG(hX&C)80ft?h;N_8x@Vjd=1~{FCRK*^+FkcP4ZChZy zrZcSRdINe3zQECwS`dFb4Yt3$1e@+XhqN1iK|#qD%pc0({8dfx&aac7`*t&!u8#%T zeFN+^=%9{!JzF;G9dBxPh1-^2#2@V+1dr5F?EU)%z+2pbrv3fgP{nKvU68=$ofpO} ztG!XodneO4`W)0pU*qG1#4#=^0SARO;rpgFtY$$TL_SZ30Etj0JAN8&w((&$*Up2d z@)#!3uY%FW`RuC7CVUgNvr)A>fDgU%99)dX!W6BytmW`9e|G6{xavK2gnu`%nz~fB z(oqN{E=l5?E0yeOw?FTCZIGKYu85cUm;-^+bNKCJ-@}R&CC=pJRk(V^7}A8!L*U+t fY|6+@dT?VRlEZBH8#SN3dGr{XV)yb=iT}a>uT`2d literal 0 HcmV?d00001 diff --git a/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json b/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json new file mode 100644 index 0000000..0f8345c --- /dev/null +++ b/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,17],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-agent.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[17,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin b/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin new file mode 100644 index 0000000000000000000000000000000000000000..82a1e3c59749280ca3e32f4bc4ead483647b3fc5 GIT binary patch literal 13828 zcmd^``Cm_87sZ<>B^orNG$K-IP~EeyGFKESA%#SQjD@74c^;Go&4WrwgT{OICn-rH zB*GVk5M@ls(9{3%{Ca;p_q_I6>%G@;D@>*vc6&ozX<~NIpu>UP%I!R}#rXX>?4o zl~(%k5t}VXHL}|Jamrok7O3(?WE65__e}c}Fd_qDF93&Q7%5aEROf$)TdUu8@dFCNR z`u3y~-PKFc_(}-b-ERTQ`Ai@<+X=*q6^O5~6irf@3p(ZUaJX#=F3y{S?|Uzx=ks}V z!>R{FKP8Apzs`emPjkzEZi#!(7Q-bM(ESLf##7hpbL-+^m&@ zW#=QHX!$t|tNliFzHR0DehhLKWIe&n(;bi3NP)?YP#R$`i!z6J#C<^sY;0SFnc)%m zcw06+Dm8-Mb@_DF+b~iW6G@aL4Z&%lH*^oH!<<}Eh;L0Iv!?uI96uZ*2Q}+y-V-D8 zXYOm#a`7N!7*~_aRmIE~k#7t=+{|n{;ZE1)XmgV1meB(O`=Q_06El39xUwyUMCMNl zH4`&Nvs^jUSvwiR%6!efl{>(u;Yb>$t4RxUi?}4+(`4hldaC^>4{PM*X+VuB-J7n) z>^90`B(FSTgtrEgPhtF+9u!E|jF#DOuMY0Rk(6)}rj!az2%f$Q(TN6rYV zLyGZ2D)o#Ho2G}uuNE;HY8^|`M(fBI=TI`5kw`5z$yf1B!IQK9x$faG}C-7dbw$#h2B)?chbeFQ}=Qi z8Y=kBHJx#q*Gz`YrD1XHB|5e1BwUFO=|9fj zbq{EI=F!iUXVHGaa@g!KKrqRRNY@#{kZUvX>b%bJ9iC46#0;5dmSqrmt(~0FY@qI? zQp~L-v1Iwy%g7&=MUJJ9)1U8ZXzd$c8n0svyEaV4J2mmRtz-$DT{#zycFurpK4#?2 zL2*=7)B%m3s`y>;HBD>T3=TWR;i1_NvY|T+*4axjH};%?@V*TAH*$ekB~QUnvw5&$ z^AqN7=u4t{poPq=&ZLWNr73rLE!6e;G21_T<9jZF%((iKKP47m@6vE^wlD+@YNOER<&g52@d@-fLKXmJaeuMS2|y({Kph=IPj7g->S`Yb(%jo%ecef<}@%G_6PX` z*O(W(A5$ycakA}|J8pZJ%(QPyr-#lcpmx_vwCT=avb;x_%O}6ldsp63MZ0-mv65@p zdc_WlC?DciElhfq59x0MQA3EwywEuUMZeC1-{I-_yS|P)DHV>HLo8HB9-{HD1Momc zDz{PYHnV}fMjphi1?{njhPYZ~^z8~KwzoIpJNHy%etFN|CO#boXzytVan{zjzA=Lf7gmGs?_Uz1m!}xt zwa>}+oo%GG_5j`+bR~vKhe1ndAI6$ZfpjrPYI*e)Y1LYT-Y*03e&lbu{kk%4`cX=6 z$i|G=6ECr&d6p!mC}4HGWG-4fOtyB*AuzX$yg>h6Q+8P z!0PN4GI^T~DK|)i>yJ)iUFFXP>jh`wT2>OV_x5KbjMJblx|{49od;GWS1?QJ5-x5w zrbYQOD1R*mE=^mEYXuj=mYh#;JZ%L;P4lPO%z3Q(7ms}p(m-v763CY{H_RFt;*Qij zC(b`psr~q8I_vQ!Jao+-<3A?Bo!Pnd9Z}|>tQwDM!RgrI5=m`~E@A7Ni>UBxIo!=U zjB3NFu*qAKH!LXwu*4QbeRntXTej0N#b>Z0@BzM@y^6#Sq~n%*t`v*|$n7y9vPX6@ zZ=ppwlM=lVJZ18+#V#Ekm2~0$`dM)LK_>C5K~Nn8Smd@1Z{8))m8lG0{exk?eFFSV zYk(0lN){j92@xlyK{4hR*`K$N{wB$^?ES*BSCn^FAOIECafe5`Vry6`Q5!Rm|H9+8gOLQ=Rycp9yF ze+d?HQs5?Xm0VG2CpR0NxK#fF}4?yKx@Sqv5#GXiyp**D+_>U#Y_$L9yVy&FM<3erx}UN z5pF}91#USijJLh?At)jkE`8cfg1!5QwUQ3(m?1(#_Ggmk?xJSOdTE$FEdmr~X=AE& z6ZO^lM9-x%AYo$(%QsJ)XPpM{KFFby%7oSrEvMT&C*!`1x%B-GH(VuLT))#i0!q_O zP+4F#_=T1*UA^bI)GyZ1vPuXO6}RKLK7G1F!x%9<79 z8n?AX!}Yw`o7jh>Xh{%#*WipD)lt+S(Gr`MPxuFo7`)Lej6%!`==kamKSoaDfO{3A zJ{U;DUk5XVR%sw!l8P%Iu7>d`)m+oiJ~%$v2m*5ZNfP&i$WG4yx3)vTFDC~l9|wuACHA$0GgwaVFkgpa5)_I$uGtCq~UMeBMe!=+P+buvt0O!@r!UJLkxS^ntvq`Qe zhu2Rg$9-pWn|%1-@q_E!*E=g8i;kD$SlQq zD2xG+Ha8M{)e}L#gP-Qdr(l7p9JH(!23}_rEPWXco`Vb-w>(CtoV3R?um;LI?~y0b z+hNH;A@;na4o$xi40^Xx=uyv9DifDP%CkS2$-I>&$=g>@FL`YyK6WX5*cxYIAE|}w zZ@eVIcl}X!tdWuh1U_33*wdy&8&&=B)AVrea-1jr5j};GTW^q${s&QW?HbJTl_ER* z!?Cl^1I!P~V2_P4#``ZrEqyy^5lP0VdvRo>dH#TNq#6bo4&p!gR zbU(qGl0p)2)t}C@Gr__sCuvmmHcT>GZQ7ijjh5pJ*bP^v@wO*7(yn4*aMyYcf6qR^ zOqVh$FJ(e%8|s-kv)tJAaeqnj>OPvJO|Z;x309cQLfiU!T=IMx2tCyX-*PQ5aMnPk zya+G)ttZ~Qj^UZ$GbFSv5Qi1`;PuxC>K*+rBr{cn{4V!r_T%mAn({ZhzlO02kXxfq1HR}aP40bX7acGzBvKYSO3VT z!qu2D=tc6)2FSpCXS(HbB2BEmOpe^x&j|~RHI#@~5uN{l4EU`=4bjzbX2o(^@s+{c zdE1aIN`v{@ZS?ksJvhfSja)xfO=i8)LGSsE_w3F?&v(tTaG z2^-YW@ZD@XZm>B7?@jseq{sxD?;PAAq$Zd5|k4_5Y_Wm-+w!Te|Y zVThzu*RMEpbX>+mQZvknFP{K}^X(+g2KGF2QP0tJ#LXf-y zeqK43ocv-11~Dmg-H8xJe)U%1TVa5|9Uqd^Xi>P}k%7hIom9Fd9M`(+!w888tUDYI zToZCr%zWu310euq2QY0-r54souuz7Dwew<7&bx>BCf|p78%?N6#cAp#w;n61j)3#$ zFgR7L2j3oT;Z!e~Kys}$@eG@e)4%N?-3dL!CC~5cb-UaEYkEyd}3GCV- zg}c(9(aOF~c<=lQhH8iE<9&89v6U*MckOBL+vtex@ypGsdIVuaYB{UAPmuQ@qZVZr z#)0skUZ~ylA4(SAMd|P?P%YdK(!W@CX3G*(cxi&FKO}J}>j%=u`e>IDmRF9S9RtaSdJ5I!M!48F7b6XJkYzbfAo0aY(5@1OTAy!ZYkfN?t>_}dvlhaW z4+c=(n~O)M%YfwGcThk|(PQa!IM?)^1Xv`)>{T9Ur?QTA&oJOU>axeI*Xk&q--%-S z=TS0!7YHs3!q((MdTehnE@;SygCX7!lax$bRs~a|0~?|9%2m!(Iubg1bUFF#5A^Fc zQP9!9#cY3QicxEdvFZSjO&xt)Vg3|qym*jo;|qfK6}yS7=_=a&PyrHUHh`qbA?S{Y zflq}QG$3V+eB*Z~HffW&_d>hq^kYGU4OPHqv0CQq%b(=d5gpX^`o=gu3_Cx5LBuQ*`ph9p8N`wsJ~6|tt@;|4j|ewrjT^8gLqhcr5%s{alc1}Veaez zq(_fqp2HDn?1?5PtqzgXa-zWHDG=qT1a8GMQ;Z%Gq`aIhaL`f@9}O(vl^u}ft+}Pm z+wxt36}Ftu`+7YW2Qy}|5vP@TUq3D3Eq$oSUa=GAy?QLg`m4&aC*Kd^A}2X6%RLjF z^k=hWiZgipCrh}=$+OwY%{35Muf=Z7(d6|toB=b?VI9vcWbbIKVqLGzWL2G)@=9K; z0Obv9uqba+^Ah&TBOTV%MGiB4jo2MWp2E{s9bROzDbJ%*hMm`P2VNOT z(w|0+fYzeC+_or4R8`~M{x^*`IR4-NJ^!&xx(r?hIe6()GTxcUD>LS)Lkm4b6K0su z#FwvVp=1T!F{cKMIxR4B>1C|vbYQ0(%Z2uoka%-_Cb|AQiT1ZZl@tDucrk=?HBKbo zdI|`R@1cW>{ZaMmED#suXOHFuAn#NOB-#wpPiA=p+zirtqI5Y|+?{+ftU(7y47+ zT74GsyKX`9@-Vq7cml`%upm05iMM3a@$4YWyZWoxRbZ{3jMi70R*Xxe}%X7UO}V9IlO< z0~u#BVQNYMWe<6QuXrClD8tc>ho9h?*9o}y%s$e~J%OAn3h+5m9Kr)1z`=9w4OeZW z$xfimCtGC@G|Qy0ath=>?*@zbJ;YXFrkO#U4W87TiucY3<6qf8=+3g{#h2cIPYvC0 z@%-h}EMS%-S!4pAX!{NS8F?@VE|s9t>puNINt8 z6msZSt`;gTR-;wIfQ9~ z{djjwofYjJqcKhA@b8-#EVGm$4W^Rl`ZW*p3$D<>8x&pG9Wds06(tjj@Z`lRZjNmv z&fB*V*G4mF@mU{*Hg%b49PwZ$$^W1Rn)cYU(3xx6>_yLa^wQ?6P)_#j3|3CMjm8YC zvb#izQpJ?yh2PhCGRB+ z%)C1dq4}c*`=GFvW_JX^v&)JocwdgRPTvh)dQtGwKp$uLZNtYxdvL71k{)U{z``f1 zVa}ONxYbyiT@*AI1*Z#v#_)PvJpK@qL&s1<TZu ztSctipE&P}bO2`kW-y~i9^GdoLw4*bI;*XOz4<~Dd;&FKpW|^%`uq&NqgJ3epAqKk zMBy^&*|=v$51tD<2|h9<`2MLaJ+tsSUB0N9#*GT0zRo8qJ98lljrBAB!{@-ibP8xk zClX(|HK<2kz}H?ol=975S8EEiNCPHzYEw9JKmmjd7!^-LCGK(=7dKuRBwo^ z)-nUnr%QPG?_hJQKE4Vy#p{QS@z9hS&`ZmQfUcReRiPAXjS|3W!oSb1ea|H1X5j+OLcCIK zRe$xyccQUogu9}k4W6Gjs5eAm`P8O-W6%RK$?NdrD5tB1r>~xdqG{2Tk@x@|_naFtneTM&-B`Gg zIv-+ZWWXfnsjT|$416`i1!Vte@iOlF(n)SN8TqaNn9V!`8LiW>Mv;MvYlb}e8SxF3 zdsWffz7i(a^7F12-y(HRogy)d%&5Vb_oIR z;K@Yw-)2%H`x%|x!tux4w>0U$N$8TPiREV_aOA`_+GBo^eALgu*jq>F$xkYHVayF% z*JU?UdZtrmVpsS@CJfIBuY_5evgBQgARc~m4oxag(eIu|u+A=kWq)Sn5;7E$n

#qy&@Qw_0lDtNb6(-PHIGVn^UJ9Ct0_dQY5N85NqRC zVfE@7;&V_L2HvrBXX+ky_qPI^WHpoRDq_)1jfLlI8$GgF8blw9^CT}uf^|V5XiriA zBkd}@cjhd}JY0?cg+`jGS?*)xe|*KgNmJo;NFH4%BTeOtlHu*G`(%YhHS{k(0>+}w z5Ykyet9F}1@&Qr!vnUX~EVS_Da1mWLaesrHZc{1qs)=2d9@^|I#$x+!ES&C6&ZVx! z4@0ZTl#B*0FeRGwZB>HSUQ4q0=_6)-&J{R5zm#5n8w|VM#8ArW9b8_r3&b_IFi#6^ z5b33l$-Xdg)P21lmfcC9&Bd4S{lp!5*z5qiy_Um+W(2r=8Q4<}m>{Z01%ub3%$p!C zwMr9gCK!-v=TgX?$pi78CD0X@tr;JoiAF=N7I3xjPO}gI|xpcIGG4*dc&! zQ~W3wbq=HAUr*$Rqxii`g%xg`2e00>()Kx<5DKotpNg$$XHbdb`(ERYj1u~2*JNJu zlyFqLybZeOJ}M?5%6=Ri=9Vsh1(QzwzzwAxRJLv|YuXipV*kppaLy;XT`HWhi*%v4 zbeF-=BO0jZ;e)%QO`s;z0-Z%hY5VFK(5GMywyBS3+Y}LS=i`WbkN|Wa5P+IIdE^a? zFsgHGaA40HTE222ZjXs4QKKIq=a3GLi1jj^ZIaYsH~`NdGba@Rv#77&a(uVAmWdH| zghgwr;I5l3G|KtIg1-ol%j#+B*Cr4zn2*ESpBR^>9?~#+AJS7EKxddP_y{c~v9lu~ zBf1@iJ7@D&xv4;bBp+LuCq>jolYozZE`07dg@uj#vFo%1wgMl|CE+ys$@d{~6oM}P zs~{9_#JirogKXS13EuxVOn0d%fD`G3=JTms`{}7LG#1=&J?B0x)pj9Q<02tBMutuC zp3cfk`O=R=o;YR1jzl?iLT26!UUBXW5L51f`{A18Hn9K&?L(0NUn0DGD8tJOK1{kU zALogiM$$Ku=deXG9uCa7ND7h#+0Zdnu==M%dz>t(LwqOPO*oF8?^gl&o{boF5&r&8 zf_2TCAj3L=&Sp)`Y_^(ljq7@_M*KdrIO+?|8i|9ow1NE035Q7gLFS3-RXF)=6C)nh z&xm;Zrh|v}L2bAtJvV;^1TM2CyPiIP!KqK6?(Hx!$Wui1=6>cj&ZW; ze9#-Ag(DG{K>Xk)9NHm7qgLhNNuLl*)9S_gcOOt)au1YlT?+!0PtnKqF7EC=4X=iu zVM^pL4AvjO;^nP$)QX3?ua*;mttaqhh$L8!+$5uVEoL@F{p5CjG9Im1g0{bNSj&M} z)GD{f_uVf@Tz@i%Ikn;E+-c;$53zXAFbKu`a@Zq=Bjl2T47q+e6#tWFX}rmC5_@3DjvDYt?Z#^Y_n=jC95$v%vbx@48LGTdp^*{ zKm8E6&7F)^q{Gmq2N3nq9=`NPn|(f!gWq~8iSA8#C<-Wt*@p7S&NvIMW80y7+Y@-3 zPzpocCfu3#*U0=ox0t?t0N%`Wp6?Ax9kV6yWA-y5c<2nYt1pF(pQo{k50yc3%Ut&G z?L-)}3#9X8E!ec6-8jZC$WDzp0`@by$VJ~esJ<=;InguFc!MbZF}h4so;GpsBcH*H z`n#yJ@EnBs{ei}@ad`Cg8+iFUfw?HsXUq zP0lf=kf#1Hg2ZRxq+E7_XI1Y2FUNgkmj6+*Th$U88(Uzi?prc`t{MD093hV@hxS-G zkg%Tu1KyfMkB$>t=s`ux2o(F{jL!}iqp^@BzR)vlINCA6mn~%2vrY?BZ$0RL-RuvFc6P{^}KN!Br=JNY9GR1N)Ege6^p5v_bg;%&A@5bgbp6)fMCZAxbv-$ z`E{rO=FBcb=|i)yR%jz>idP^>{B?}aqbE$`sv^>FqchRRnAjoLtMH6yCafPx0{*VK zSbCIYemVL>PPREP%jfe9vm-#~QyUIu{%*Kjs=$k*J0a0ch3L!2vFT5vSsq2Y{!er1)%b}Z-f6{=* zHqckx0}59UG;DGmfxUMP*z*SMjO5#B*p{n@`;~>+!is$2uq=|iKNk&Z+val>f8Ls@ z2Ww#4Y5|CskV7S%LsUF?3>`k50+-wl81%RX(mP(k^&e#*5f#bY|E_^{Vb?ICa{wch zgJ}DQ5S&zF3m=s%z?NB0#tnK=DJ>i)F{8AqCX{S=yA;X72~V^r7*1RB@X28@qP)cx z8Szwnd0z?z)%ZxSS_xTqRE4hB_a$CANu0{*Sn|+a8BNVA&`kd~efL!yc?S2< zq`DI4uQJ9(x@kmlc`h^B=7nbk7Sa0$CUUpTIU=boO}zTfVa@#`=zX`1irPFSR)T!I zq|I;1tDtw*f9}xPjMWd*N*JcXILDzlowqJG^zV$Ki5yx;3B?4%NMZfd|iE zd~-Cgm!8nc!eNZAL@^w_rh;|-pWx^HD0u#QI+#c~&;XAoaQoqFu<1XG(S8Q_b6gr0 zozMWsS*EBPw+D75rIPiEX;Ahq+{{_80CF1=>FM76X8aT1v-s9hcl`)h9FoUb4yl6r z*IN@i;3Tf@yfUt`2*QYUZ|K>7FX)^FNnC|`CcbQ*2c{Ffi&{mhXSj&sKG}!g;uteqU?|7+>7&yL^KI8SY8>qQ7`md z04maK;g5?tw^_WnZ=_&R)Gu`jHh*FtT-C;es6GLg}n;S5Y6N6&{t!^uua=Mw_=`HrByCW___(U{Aj@+#}}I( zR_nl$3D>Fj=MI!TzYptrXL08*H4)wDqa;As8k8LMVb<+7On4Q5ap%{7pu~5YM;5`=`<%)-w;by70*1q73*VuSYERmw@No zXISvxHp~c30KEoJlAG#B8`HKzQM)Lstrf=A7e;fTMtN}2SA-hPc?QR{Rj5!{5Ki-y zWal5rh zYv`vpqxASmeV%8TB~R8ZAG2O);Gz!(Jckx7o^_uT`+PzR4;#$nS=pH5%TB@^Yyn#O zL>zuzFJ@MYt_5WuJ-F&E4xS5L*yh|yD)ZI_56*f*iMZ$R~d5KYiPRGO+=>xIB%1Q#@|C| zrmrN{e+Wdeq6i#f)9Jo<_i4cs2Wa{*3Zty!yNPHP`3IzX?(c`RC%?abtIDLu4*Uai)wKHoVT$5 zT_4%+eF=(pc|aPih3X-15WchlSIi$`uD#>Y$*&H;)zJU&qvAQZdHXul?#_fZo*Vfr z?Ff1=`^dQHYBaSiA;+ZWF^5zPiBs-53c4dR6sWF36 zOYwUCPpJRvjoY%V=$xH4Y;yfZjxSspcWiD0<(PY5J zsBf@8kjAym-bmR$hd}9tIkd}3!^~U!==P=CYx%k+{uSp!@R(?n-qgxA{|HV6+$tT4iBnV-fyNj>2Doow#fH zBwpRdb~;C=gxp)ln-_Vz9f^D*+XyGQyk}Y5y`$ps$h4g!D8<$ z-nP@PnN^R@(|CbL$Te>OyGK>T^1@+~zBCR>2H!$J#D9?Ist#Hmd9V=5sriLJw4*hU z+&K{r7xu}sdY6+()4@va%-t}U5qTD7zXjgCkq_j*qx!7&$6{i0I~G1Jw*tPjuVntb zVEp4eoxCh5!ZL?-WRJE8&*W(~6_4egcyFPG4h5z(*&&qsSZsiE-Solwr#!TD-=?D- zQ(3ZU9rVBF=i!@e*T2S!V&z@QH4jSAk#t$)RI6KdGva#ssZe z^w}1GEw8drZ)_*oF~QSSTtf-JfDgHMAQErg&L!dP;TYLoj?dcwKa5tBM$2;$QBy*G zU7GMcKSL-PAEb&tZ_V89{2?W>sVG!j40gW?ptC=V>0cqwX?cY}rJp<~*uBM~-74_w zgcnbu{uK#VdqgfzQXo$>G9c408*cQp)0jR{o*@4VVke=+dt|PH9e*z1l9eloR;&R% z;9`jS^Pba=!%t|9fHsP*Ekj-9Ul7#395&7qWi32Psqpqvth(z%e^>Yu_k)h8l@bDN zMiwyJ;Tp|d+yaG%U9iJs9tl;-#i+%fsa%gOs-Fien|z)+6jn8?zH<*J%~uBBIU6BR z`#X`UUyqklL(ywaGWPhb#1nlonY2ezpVsiSuXIbx*GQFo=kI9G~oV=4!X6f2s#tJ@#0QCrY=Gu8md_?+Q={~(vS_XqBZy$jEt9pk<2XhE}w(&%}7HIc2`0$EC8P%?I%$j$F1 z#{|X5$4h1OSY;3maYt(JK1NH0-;8zf~P#}v=-renGe7&+rFtyQ{5qelG5 z{>9@|tu`DCL?z(mEg8J`pEh&#_A;}SlqM=1R0KPkiz?tqArtXm{ubwA@ a{aSgf$gLQxtJ=c;i#Z1RK~6y3#NdD4a$&Oo literal 0 HcmV?d00001 From 0345904aca72e6b9305d6ba65cf6fa7969975f18 Mon Sep 17 00:00:00 2001 From: Thibault Neveu Date: Tue, 26 Jun 2018 19:16:56 +0100 Subject: [PATCH 6/9] Prioritized Experience Replay --- demo/webapp/public/img/ddpg.png | Bin 0 -> 32519 bytes demo/webapp/public/js/DDPG/memory.js | 83 ------ demo/webapp/public/js/{DDPG => ddpg}/ddpg.js | 94 ++----- .../public/js/{DDPG => ddpg}/ddpg_agent.js | 114 ++++----- demo/webapp/public/js/{DDPG => ddpg}/index.js | 57 ++--- demo/webapp/public/js/ddpg/memory.js | 240 ++++++++++++++++++ .../webapp/public/js/{DDPG => ddpg}/models.js | 85 +++---- demo/webapp/public/js/{DDPG => ddpg}/noise.js | 0 .../index.js} | 2 +- .../{ => policy_monte_carlo}/policy_agent.js | 7 +- .../public/js/{level0.js => q_table/index.js} | 4 +- .../public/js/{ => q_table}/q_table_agent.js | 6 +- demo/webapp/public/js/traffic/index.js | 67 +++++ .../actor-model-ddpg-road.json} | 2 +- .../actor-model-ddpg-road.weights.bin | Bin 0 -> 15496 bytes .../critic-model-ddpg-road.json} | 2 +- .../critic-model-ddpg-road.weights.bin | Bin 0 -> 16132 bytes .../actor-model-ddpg-traffic.json | 1 + .../actor-model-ddpg-traffic.weights.bin | Bin 0 -> 21640 bytes .../critic-model-ddpg-traffic.json | 1 + .../critic-model-ddpg-traffic.weights.bin | Bin 0 -> 22276 bytes .../ddpg/actor-model-ddpg-agent.weights.bin | Bin 13192 -> 0 bytes .../ddpg/critic-model-ddpg-agent.weights.bin | Bin 13828 -> 0 bytes 23 files changed, 457 insertions(+), 308 deletions(-) create mode 100644 demo/webapp/public/img/ddpg.png delete mode 100644 demo/webapp/public/js/DDPG/memory.js rename demo/webapp/public/js/{DDPG => ddpg}/ddpg.js (74%) rename demo/webapp/public/js/{DDPG => ddpg}/ddpg_agent.js (66%) rename demo/webapp/public/js/{DDPG => ddpg}/index.js (56%) create mode 100644 demo/webapp/public/js/ddpg/memory.js rename demo/webapp/public/js/{DDPG => ddpg}/models.js (80%) rename demo/webapp/public/js/{DDPG => ddpg}/noise.js (100%) rename demo/webapp/public/js/{level1.js => policy_monte_carlo/index.js} (96%) rename demo/webapp/public/js/{ => policy_monte_carlo}/policy_agent.js (96%) rename demo/webapp/public/js/{level0.js => q_table/index.js} (92%) rename demo/webapp/public/js/{ => q_table}/q_table_agent.js (96%) create mode 100644 demo/webapp/public/js/traffic/index.js rename demo/webapp/public/models/{ddpg/actor-model-ddpg-agent.json => ddpg-road/actor-model-ddpg-road.json} (79%) create mode 100644 demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.weights.bin rename demo/webapp/public/models/{ddpg/critic-model-ddpg-agent.json => ddpg-road/critic-model-ddpg-road.json} (80%) create mode 100644 demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.weights.bin create mode 100644 demo/webapp/public/models/ddpg-traffic/actor-model-ddpg-traffic.json create mode 100644 demo/webapp/public/models/ddpg-traffic/actor-model-ddpg-traffic.weights.bin create mode 100644 demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.json create mode 100644 demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.weights.bin delete mode 100644 demo/webapp/public/models/ddpg/actor-model-ddpg-agent.weights.bin delete mode 100644 demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin diff --git a/demo/webapp/public/img/ddpg.png b/demo/webapp/public/img/ddpg.png new file mode 100644 index 0000000000000000000000000000000000000000..130f3c978b80bef4132ba1602c751a41ac5eea2a GIT binary patch literal 32519 zcmaI6WmFtd(=9x>1P|`+KDfKPySoOL!JXhP!8HUAFt`T^F2NmwTY%tpC(rY~_gnYR z_oLTL_v${~bxxgKyQ+4qnu;tcG9fYm06>+OlhObHpspc@3L-S*%Jnm1EaV5qLqc8) z5fO21M|B%=O5iE2=c(yxPWu*|vrO9K{km9G#$q~CZW6Ot&9g!o2@ew9m$oS~`ECn8t?7VG7 z*Njb##mC1BRdbF9!#i3@KU_t2j^L4AkYfHlW$XC>AKq_?k;kkcUwi!(hb` zaRD)3yHavuH1`DT+8R)Pd41Pf(f zQrw+-XPL68n=i9L(L@7h=M%>cY^ai*T*&Fi6kIt8YGU2gKXHqR`KD8JHu-O4iT&*9 zkjZHlV^E9ZbwOtMee~<4`|=8Q@zUsVvB^}g$Ln#O%Jn(v{nT}BvwiYh&@rw)9hrt= z;CZ`7&yc*BCI9=cwy~H22Q$coA0bZ+FQV4=P4Mu`#jEDO=d84vG$VIzy><3JU3~}+ zrd0PoU#oxnyLb7VWBMT0@kyusWNHAtT)7Ct*4DV-EXSA_sq@w^FJfn`Wo-tFNk}5IM^Gb;y8|OPA|uU8e6ypi*ioR6>5*EDMn!`4 zUCsz0V@t@#qzYx=E)_hF4OcI{Kfn9h*&l8d`mgl{d!e>wHU2%<_}2C?MYs54fwv;u zTK0ee0|g@#23`zf#=tRkps8Fg&RJfO@Xeeoqn~}bVcNm?T;%@S@|d7K5bRu-I~wdmnI9@|#PXrFvvzBf*gv zQjs%EvXlGrLcVO!n|IaPtv?brSs)3<&_cYHvXVFCc>b2nhWqnlsr+;<$n)aNlJdC~ z>dS~SY81~b@|z0=bFBedjM&_A2L(^6wYfw1JKkg#?fbbjSY}@Hc$~NQ5pUINO7w2I zH|TOLN#rjWmHX*6{mbAOYrU^r{86kJ9L*S9&9bd2oB_h;7Q~2*mM+l7!msK_4RzZD z$i_bJ3z=dHt}X)#CL&Pup_~Gnrb)rv`kiMr6dyo}6RyxH;- z=0`(JmSV7l?0ek(opg7fPBnUVk9MHRMG{jv^Z2)}+9fk~&lE!2dp1{CHj@%cW}--F zdiKIu_Gal@F$KyLIoh~Ecdx4((dGY8!)fh*niPk`~b3? z5)vvP{KBv>j00`cf5;8{8@l~Emls~p-2XpFe)M3})??#{NTfjUA4nIMal8#)e&@~f zDT4*Zm7xH%D8nknv9=8*@%Oxr!@KL1u zMMevx!-{=;e7x^{hrweLVJ&xJv!--&KWomGJ8tp%uzgfdrH#f|vs~2?bvLNl{TE*P z;u?6P)`JqR>tdIgUt!!NIdT(hN>}U>OWSR9OAR9= ze@Eh(Je+U6oo{Sx@a$qC@w>ENb{1X=?>}o>o+>+5%jyozSXuIrl=6$nc;x*M9I;DG z^F8z|d8EHNRt@eDwnZ>2ul+4J_3e3@_v}wJuJY@VCe0_$AZ@4;tg{A@c$-pwO1#o` zS_QkX!P@q66@NU?->Wo>1J&TCd4p3P4|nOOD~4HxpPBDYz|MyBQch%Ttfsg4gP1bN z5iihhmne5K!_BMA^UDv!tMf7C`{EOm>7xSZCssdc>qU?uX9?H=y!1;~F(--^E8jKz zw0G8ti)d2D!!dWQ$ivo^D$+IR=8HlV*aL+i!d{CwA5V?Gn4rag3}Vfh1#*na4?CA! zV+P$_h)w3LH#>&xWUi!x9cb!6*HeM${$ea!dM09`z$VLKPL zD;~fSHos!uEmhkI$oA;?#|KyDe|ACa6WCUc3_rI(|KqhAL3FdVs5?y+){ zb}s3P)%cULrXDT9I$!LY4|($60YeYh$vRF%FfHP}XCl#CKCkQchgEeipN`LmUG zGMO6)Wb?HeO#bqu=$c(Gh|(D(nmo6+~aL{mKSik`MLbFOFvyUYlPKl0+v%z2|!*hKJyq-scSw@%ZX1q-jW zI6hFa1LJZ7ohw#>xeA=Qonf=(h$gatU>0h-|hl^((2P!<4nPE<73Vd3K?&b zGj?c`C8tP)lg0YYIQ2E(b|OxS=OWK6SN_9nxrnbhLmXg){~9K^m?-_@!clt?+3lC- z_86sj-dpSaf?ebPAE77bbJtRz5gd3VN#;QKG;aFlZq3rYXyCYL0KB0Sxr(iD6McW2{+NOxfruqBBSQ9i zB>QE!Hs(ja!^1X$=rWfJx9Q6+UXDJ)zVDB3e(AXRnZNN zhXVKQQ#s`-G$kpC5Pp}x?$%lhq1 zD5CrLYS?>gs5}^r9J4+Bb~e3R`PezGsuN_$f=a>tH9Onw{*6quGdDo6QU%EC60i8FfKXus+371Nc5(m3tb9TX))ECD&y8e$JSW3E9eqoM?PqPg_Z=p4r zU2N}#a1VTxLDlSHAA#N3m2CM^A9>R3k1k1i;LYfa$MuHo_|Bf2 z)Nt^X{%St$yT4Pn-^1{}n;8#CwEymu^2cJ*=$=J1en4QhI2$@|!J?lt~nO!fGN^Cl)DA*jT+B+#bc{*-kGKrzA&^9dk zsgNzQW#VPc+Jico-^KLUz~z&jKf8{g1iC74PWpai8x}bFAAqb7_Th~~RHZ8^@b{OJF`jNMt{rj2U^&F2Q&>yrvvXni zJi8th!c&4j>5OPcr9g+5#5b%^ives=gUbIB5L3(BF{;bs?UP=sZo;x}!6o|}D?KhGVp#`3=yVB3EP&BdX|3tK1e+#7OX;)|&xxld+H z{PT0^V~%Y>mC@rrxI&FMkf|b=OmIh9)9a+5{G{w4tKVclonUudlnej1y40id_*#q} zNH0+JK)cz##~aMysvV-j`B?$n^Veci#!JzI<~6$|2%D?*^Q=*!FB>0nt}=*q{?wW! zP>*Mxw2@TS0fYL2rWK3%?|`AaXi3a!n>PW3VC~WGd@)p*u}e%bemKa)I!igAIvO=5 z&WttL#%}>K2RsZnVAa$XpPUIW{JG=OP1k)BddfZJC7hfkCWBxunMciZ&|On>Z87(l zWh7>6&eal&HecYA=YH#<-)!iqq4|%@>m=I(KUx9h!N#HH!?&3@cNf7=^|80tfd8Zl z$S|&_ucjYY)2ZlxveOgy4|~AUcuipnePvZ3^!*1o@RHohJ=XqK_D?7hilL$}JLn^- zj5eb>Fhq&K3gP=`OO2^;hf~Z59T`v5AydZ%Z=>*H;eT6eP-{xk?i4bE5+sD(aTfmK zi(8LhstbcJMuL0a_LlO=Vfx&Mr`PYNMhYIE?Q=KIce~P924l zJwztG?S_*HrR33xNZdR`@u38QL?}@#_|d~_m$E>mxptjVgbT`fL*{`Yv5Mk5Et6!X zJukb9A4Ic76Rte6vm>z$nd*c0f`j&@y>?Aha4Yl3x}GU3Y|QD@CwA|u>wB-KgMDny zdJ2UF_kT*gTN6oeoVSajxcz%U>koEr%^`91{ie66#9l8ZpY!Lep1|AAt&V$hCDXnm z>M2p^HK<uRnh9p zL+e7Y$txl0ZSQVI4-9gl5~Tf%F-6>Al$?fbe_%~sF7}7=*EC~4roy*zpg@hZ3*V>h zsZB|`)3$PK%=#sA@OqOBz4@E(UyrGp1m1#p>t8>4yy{dn6l{bFMm>%c!5|_?la)U2 z-01VMLW?2tv1a$9)IyOB$khEg@d^4fxuMxvS$UEFCI4of7M+IhsjtytI(oDr^Co%O zpUPiLGJ1I(=A6x{?#*|n9k2KK-q3L=z>$CzF*~pCIC|R`V78_g@NtN_!iIuH?!!bf zmPC9AAZoEwA_OrrI$`Mcl-JbL*hkj&VCePG=0Z{6cu+tFw^=X?=Ht*K?{c5`bDf{&-$4zVF^#kQ}OZht}7z42v0*YxS6Deh-dCF@l8I7$l{^2j$~ zGRi^YslmJJkZkD{EaRWeOxRlH=BKNx=har>h}0CYPIxX@j3LSU1$_7z&~uw{+Aawb z-wtZOO3k^+%uzAeu)TD6Q}lSA64ZhmyU96!VDk`V<|GP;?SgG!3RibhD1z(KDa)OmB2s};{{sh z(#(Yv+}ZjUZkdK~i4YKT$9gEG!7zlv4%SV{?dyGpeLLxWYkfZ);mx1taY8iL5eCQG zPcPtaAx?H2XR6?*5}iLW+E=Ca{4HnG4idfnrFrEMzd2_OeIWFc_!Pwso4PoV9*&{0 zGwK(0nHYwcf|!_`6Dw8ghqXvF_F%S(%8|ep>GWo1)zvw+Cn*pa4S85{gOe362Y=F| zrk)W!wbL@y0BMR9f$b&lNQp2RR8&h|v;zih^Qb`w%V!lK%Br9|7_9XD^|<;qSo3hi z0SPr|p!=F$&u0rXi9S{~H7mIf?Z!21mRGc39&!?fvQvNia(5pWW`>p_?P)S?8=Y7) zLBwluhSJYpLJJp0w$DULQ2an5oG7OJ=<=@J66M*HHm#+tty(j$POnnfXHKz3A{M(kfzEJqICDUfyFv4qwVM=+(J{Y2PWgK*47IFn6+0jU8si#?;}l*5 z)`9P8^db1iZm>{#gWknBF5PJ6JHOyyrTLKZgb>z-AF#6&KskJ@}r}os9RiO(}ffY|E?Z}4wy3?LH`FmEVZY?ZBOl9 z%|k2T%Ep1QSJ16eKAIn?54jWr)MqmqD7|5Az!zH#U5hpla2xapr^ z9BC51FZ^%BvhL89S2#WhwU{6LNUruHWRu7c50>Rlhy(zPfCJP}FgFaJJC2@_=R~D1 z>xtVC-S^-OEieEq(=1P!zWCrSGAr88JXkTNC=A?pYh9m(uS6E6gMmGHN+IINl5R5S zfRQ?3GAv2ce>0geLR$@Pey7$8L_YuKz*RM8K7gP(4rdU`4I7H>L^PoWRR*`1%zAD3 z=YyQG7(#=SUBZGiA`~D9X}$?R|Hr_j5P*_j7)Sxj#&SkI@9$tUch3zBPubeb)woI3#NA0=YqdTAjb(`))Xa z7$Aql2Dbj-W=I3FXoOAdyI@nu|s8pva>6Pf)0}~0aZn0t*eEO+Fr+| zBdjeNwOwljJ<3>|VPd4*<&fAEu5b>r3;*^7AHI1gjrplLcPlAlN@26-d(dGvROr5yhxxcjdQJDRpn@V^k2_5qfy=wLsZ=9?y@guKt)b5$>gCp=gh2E^sEi{Qxo1J$}p& z!B(xKplsBv6Xik(=E9o$ndLaeDrvV6#NhH21WP|w)y|T3-FAF}~b$!2ila3QB` z<~|?qg~BX?F6S$LY=3XhVll`Pt%f>N#chOlbZ+-{__ew+Vf5Onl`^+?Kgpm2ZG@&G zWre=*EP@&hjZhO;15P(!`&tX#)N-?X{@BEK%Z}Wc{UsW}NtBF*$GRI@7==y$8?)vj zkcXU}vMI7v2$Bv|#|gGpV$n_5cAXGZVDTg4oo^=2V}=@0vrgf71G^d8sW9DiG)q`%;_w{?yUoW$YZPKz~Nb@&7|N!7yyg4^!wp?I4Hy zzrVUYnmkM%p%T3D!S2I-t551*W7Q-gz1pXKx9#^_zU`vBz`3qdkeOfwL1m(@NvcI8 ze@dg@FJf75N3qG{i);7z^ZGi~LCes5Y4}$6gT5Ia^HJLAw?bU?qr`AaDs+KX%>^lH z%DxzFeC>UV(ZYzmqP zd?+ZCTA%Sk*aji1tgF?5xVqDeqsgwe_w|WD`O0ITg(Pu)7{t>`z!@fZKP z*R-U;L7XEuMCeJ|`ktJguAtS*NSVA0+OA7Ln$RgFZsMP_oGneSvAv?q3iV=fFymG=CO>lwq&p%hS=I;F^Q`(IXJ8PEJ%L!%$y4hs0b` zZ`2kb1}uY1HAAH!7lsw#!Asc)%YsF+(&;8=ONS2Sz2qy;^ZfXky1J&@ZE)(`AU2xz zXNO>wf)us1A4NiOC5ju!#`Jf6%^!=+oK3?l>HkZsL1cSgX?%WYVHYC%s?_2bR#0ZR z6RG;R?vM9D*e1|HZhrg2*B?v+kE^FyX+;Y<`0nfYB+n&{rmW!?7Kt*vK?{ElgWvEn zwq`H=?9zh*TV^-KyouZ`#9=|}?qnjyPjA(UQOzlYvJ5b_W}VM$=6HWTWD7<9^mD4W zcqA<9m-gG;EJ^S39<&~w`X;UM{a~O%OKJ0wAxFVsokSZC+%P9W3!B4{UBIz^uFv$) zDc;cRz)!yVT#)6tLt6~3=iN?+HkrJ8L526s&uF2);X#v{2;B8sy>C+DyQiNL{>=7j zy7hk0s#@?elcRYIqFfwC@!{9S?*g_2_;`OBH(piL)S9jP_6haYp)(G&g|8gXh`Jw+ z)1~gq03e6L!hjY>QlQ|6=Y^s}5`?1s%~LxORm+7bF1IMM56sLXRTSZJRZe8zFdmf) zvkE(lNKc)phVk#8X|;v-G)XpMo>M&~;-5>A&@W59H*;U!bEj=HG|XkWe(xJi7Tqn5d!FLTkGYs^j(vN& z$#nSsvxD9AWfRR0Bc%S`Kyu%7Wblr$?iFzb6%i^mnbnqu2I!PEp6{PQ#h?54LbNZ8OcY(@{=Y0vQ&n3>G%x#{s8% zXM^z(;O5@K-+x`#P)dIMx{T$uM78FkVt*%INg4l&@l`BeblVsKM%YZ^++R zT{bZ^G#t0$EPmsg*f$yyyIV9AKC9TNzAVm#cR6-fkRmY+(sx9}tow|}McE}gjXffx z*!hLtCWb=43Ggiavw=G%SxMaffkuo~OB8KcQ}yI@!1H?&6CCdxjxn2B&x*AQU^yh(pgP?TaI> zIE+ykR^J0`Lfdb-{$zlBRS_WlSxj4TK^~d*<~E+xgLt*+@8}m8Jac_|Y_a&id|`kZ z^UfLMR+H$4Pbbh}lzFf!1>w$?0mu`}D+3w2DdW%~mc_|K%jsFu&f1ltoMB9&KL!*p zs4mB!=^b#anpT{6PW?=z&)w(`k;4TL=VOjQlqVrP-%qh7plifYH%GtwkU4zxvr0fV zN5YblYCtS45D_&Ma1*Fk`>WnC_x9pPsH&iw3Q%l-ZQ5Ohm6yW;W5~-N%o9Y9eQLJl z*p9j{HlMO7+C5Vsg^(zl7;vK(CsbT ztCUk&;Qy_g__Wu3Z9{<)IxDLM18U;4z6?q3r=J8FK{~o6S+i8oP>!gm11c*+;~$V< z+mkiy#Imws*W?+xx`QGJg7}%EYK0LTE?=c#X%WM4K?P_b1Ow23MGRcMy*q9z?_?!O z1*aPFrUcN|=bSVEy9aWs>-Jc6gA==@H)-kKA4R#O9p}p*<(Fryb(^DkYwlhW4CP*F zK8a|C{Fgu!u!-hJ=k?WQ5x}&^nn}c(Eon~R?*^K`5fqNo06siBxJ3+D zjRt3SNw{QOJ;{zy*M2r~ZYYwz@5S6Yc=tW%ds*zlakKJP-T*PgydDIJ?NvwoFMU;@ zZM-|39@|BA`Xav6Ud@gR{+}Geq#YsM2WCXC1gC9@dXT6AZ&+WEuvSvR#d5Kk(5F?7 zE)MvBTciI`7IjTtUyxEoDptzE+dDHR0>iqcAWN=X1KiEafYKVo%OhgOVGVTi&1+zSBA`8cYv7hWbh^-R5ZGJT%ctA zi;9Jm?JSpz5`nX%6aa~0JPF>ucGV6e(^^?Yg%(SG=@u*2iZiE~Szq1I`-SzCr+HKj zY|D}FxfN8XQ1MC}QydCoYDLk;P_x-ryL4x2`y)e`Tmlk^;EIHN>D3vau(e8eiyvE ziO}#g$ka|=-7>ZBUarA#*3C4iKZO(ZQgad6eaE_zt^a`{23Gw`;N6%ZGujnB$~i~h zx&Hr3ZN#(-%aTVX^z(uMiB0b`^(Vf#S|O2+dIn-P2&dz1K}elz4dZO5IONcf0>%L& zG+ERA;M5;Ej`DO?+_)U23L*PnSTIIYfG8U$;pQ%XgP7tyi((>>xOKofdpe3OXbR4K zpW7A18+rl+EHMx_z|WVd0+e*tyVmrftch=19>dE?ILd|XL5H9f5ddw?1lt1PvVua& z4fVPc+d&r~yC++yho0WrS^W)MeE`hP|~15SnfmOhvPFk0y78rn|nxM^1cga zV0aFP*;KKx=g+e6&u9RB`OV|Y)06)@8W27ZgPpqj_MutVK^E-jl=qqq#5m8%c_m5M zCt>IhXke57^ii)H%HHxCTkIKx1Y-NH_EqRasae#YstAUUhgPL1T^7U4AC0N?YL3@T zIX_o`|1hRo^$TNp=pX(0O#B%4I`+QqTVzO!+19&cklA2z)hy;GJJ|O7oQ6_aGdz@% z88lEX1cw<99CymLGt6Wr!_j-s?QqmR^}Gp^$I;8!>O<23NHTo|xL}peFYEQAzoYp% z-u;~xI#C-~igX{&q!DI&Ib9gkn*tG+Zu&>VEw#FNgj1lw{3?hQW5vQdMD8TmTwVt4 zY7MZ7XPMENfglTHLsH)hP&B)PHbT!tVCaK(e-|6qg|?@l4~1XemE8_eANXFWn=OqWN}HIh^={t7^xWlj~eR(eUE=g6(Zv z=g%<7^$yJrp4m!+Fsw8p5^t;q+Uh|5xY=-QQ??+&R((JS=3JQRzy-X9<&20Nd1rX7 zHe%=*tqh&b!3e34rLmkvGqEIMiW04?tnA(>QLg*&_Nu?)&<58Z-QK5#u}k#ehrwC- zmcIskdyxJPaoU|zIm=^WlVM)pscuV@wItlAQ=gkT$gGbD4xbUgRAUmYlPv;weAewm z-W0p=W%P3UuOz-`D0v6~%ifA7MSpno;cWd^IO|1^Wqu6vxWjSn5%&lY$@WpE2VF`u zJdoT4pEl7G6R(7It%1je+b4N%EtSfL#kYN-WCd68P#4Q5jT(}J{n#ynr(o2(Vuu`VmnBkG! zDu^{pGQ1e9RNM+)m`&!<*rT^D!g2C>r#XWTf=a7o=TAD$e%5^2!n-&-KUi0x08L^o^+-_A)M z$dOz<->qk%A*ES*aR%xUHHg|Y3`YUZiA6$h2C5A8I|g>jMyQuRZQyiORn83q2jkt6 zgaU)CjLh)xuv6y+RHbDfq-IzzIG{XVkB-q;{<3#mS(vEA?qFNWbz0>qhIS|A0HWD= zHS!l;xc&xF0)~_H-iB|IYcq9};FL(Be+J9Ztc;CJL+4*={Z0}Q3XH7U7 z=f048)IE!%F*U3E2W`Bav6IzPim(jRox|+l^YwMo2~1Uwf7GH}>*wB=6hO%z5!erW zw!_yzk4Qnq#1A{ru!!vBV~VTJeIu(RzDUUlMX;1@@^1B4`&#GT% z&0b@eKVLENbChU5#{R%0wm(LPEXckNXqgcW;~Wl?&W~R!@a%ep$m58m*;FS#sH+XZ zZqM>&0gOc#&>X_&<`^@2|H1S@f(8Hto_^cfNe4Rocr#IdG=95Z3Vz+&mA-Blf7o^VVgi1-RsYV!tG=s>c<(@8_6fU;Ltdi9I zT-(c?)i!ORm<${Hz4C*2;$GRj{Ti5x3q8WMd9rWqsZV`A+rrzaGsV?|BVmtcc3suN z1cs7Jk)iqETLE*%WVd}!0_MUSTG0Vf!v^Ca)iRF+gDFx@%j$cVh1?T#EP~Tz4+E@b zJ0xs4Fhz!be)|xc|;)1d+D;pd9XT597LLz0zV*d z4jMe-X@qmsf`#k<*2Rg5f9vUPvhn9--*9F|hmg+p*vfzpp#DgX!-Q0snv0iF4t!ahq_|a>6Y?gtFqWtp=$lP$AaF*R6X*;`#(!x}0FA*t$M9`)}$X*vWfb z9@W{LG2w-cO_37T9H-mA-en4PpS!!P@)-DCmlv1GC|Fb~{g6ntv5_hWkZRD>`KkU5 z5ip|!Yn~zkEZ5wlw|95qfOkh5wT3b@*r(uRC_n9pl^XEcxVEyVGE6E$-HOqla(>{*Sw;=oU9&YP8x^6F}qixTFJ>v;_t*gj&^lsT@tF^E8O#UJhn|<r`vLVPw|1bSsLY)6E`=(4k z6cvO0E>HU-_9I>MVx)vPA_qTzjzamEqG^ZVi~CW}Ec~F^>FxgJSK8IOVi_;Z-fmQ? z>e#tb)=!G8^IR#Ri!Y5yfi*Vc{TbAtciCSw#ZK!K?Zj*Db;_Z!<69t1*dbzo&S0!Q zW!pR$h~w^mLW##4KBoduo&bn1e;Q_CVyUdG%xQno@6=*kIKd-E_{F=+jso7qiWc0! zQ?QEWeev$RsOwVKWnvsri2>GVu~i88R%3O1<)52Ww>;wB@7n5TyZol-H~5_4Vvn}w zCABszPD2ij$lOPWp*}^L+wzy*Z2($r6$usyU~(2L#pTxu`ZWr6uJx+#9RtAzQiVYU zZTWkO-xm{hFclqKh^RE&d3)V=otHG+Vf|MPvsQX`*7d$&G6#*>NZLr5LOJsUbAbK#A`w6=>{ym>C6llpCn zvM$#*!U1iAC|~YJW}V~Pbyn}hGr-KIHYcZLai9ClVCl^;Q~j;Z=eAeM;(fYP@+4#D z_Fpn(n2XTptaUiFPnYKT#kTw}J%{Zj-79m?I)ZGkg&fym3BUe!jZ0vqBT_rxp_Km&b} zt{aj3q7d?orvLA=K%$=nqvF{*A1eL%TJ~y)XuUoafgise)*bOraj^`_?5}%DQL2o& z&Baz#F0j9rb^4DDMGg8^N^`rZEU@Ovcsg7KQi+6b7|O<@fxJe?E;9`r^1i!h@nWi1 z^*iAY1trs=E#yx=b>D9V2j3P#blLW~W_IazlX$E~UpKuUulJ9bNqx6Nb+n~145rM> zXB#MUrBRMYmm-`N2$Ce$j;^wm*ydb9vwLR2_C>(0qh#@x+Y`*YTtbO>-YznDaeuThkT0Du;$G9}i;HxUcvQpVmNpYO=TzfH6L^;yd$iawTKJGvzqRCg?N zlQHfPiJ0r;$bk$;=-^N;D|u4^9(pxwC88< zf9!nLw8r~fCV9A_)VlZfacqZR1P96;x~#hp#H z4>(pd{q~aBep0m*WnvTJOkzHjIBHFo&$V(imvr~{2~rM2JMZbH*zkwQrMa33dkk|W zNA4{hj&vTNO4O4YB(?%d4N*#JJ#Ll!+x1Q+Nz5~53uhaTlnC<^t{}P4_3y=1(_1I2 zW;m_p7J*QCDc7#6;r{-FRRFi)rte;YoiL_bwGTJrVST_p6LFNre&osLoLiCoEdb;A z!Z=0>Mrur1jp1h7begKj3d`iHJyAk4YGY+qEITUKT>_X+2o&fj5(7>ow3W?nFV)v> zDfHQ6wAXs(4kqiT$f}U$Qp#l#UioZIw$3`Hxa9cM#aUF@HXm8m!Qd?~1wggc>D#M5!UsWq){Bn5Exe4)P@%Bbh_*_@yVkeqmp!u z>4!~9E~~O2*6a67!Y4I{I#`UbbjDydhPtS+li`b}J@U*Ov6&5P?C!6nb!Gg%U=yF8 zwdoS2_FoU|8MOl!oDB`S%NCCYe+mWAIS~C5F_j!pDYi~@uks31#@!0~f)k@!G>qvt zf=O%a3569`W~>VWu7Aon5~J67I<>iU&;50P-#V2bOhm?XWSVga^a}FkEr>s-UkPo# z^;j&gs7u5)aoJ%UJ+``_~vM<1~vzl5R&oY!-qZ0y9VlcZGKq87cxw`DHE$6+UO)W zH=(v;$2Xv}I$z&Rr1B3V9f3;(HT&FS15;dLIfxkw0`FFyY*WUjNzX-t%f1y#e-ldd z{1Z#v{@!^lp8x|u3?%;f$v2{)K-L_$Y`ll#4|p>slNj95`ZfWa=0bh{JFm)%swp3x zqgp6pyz&Oo+nvGI`qEPG+wIupi9!%J=1ShJYCTelfaCdvzKg-@fHCm?X&A9Q)!%AdO zePww7LW=n87E68ttMu?<5x8h_FH=|Aemgge-04o!~7S1ylQb?W#L@ z%M(;iE>vN~2u|=%-Pg;l$Na6T(T6nA$F!LBQpd7`7=r~Rgu>;sjXk-9RY}l3u`xJ- zOLt$-Z(6+eTIL(!@yopE>-@}lF)`-W!*4p#6|-q|{I(CGl_K};(?hBy9eD`#t($RI z^DbUrC0klJA{Y8*EQdFqgBAI84%hg(k0Ub0Ugxc5c}Rfu+3INbZe~1!icaoW{trUl>5ETXjzrX>`Q{6Vtmk&^^$%_tiBgbK+EDbc;BIVE}A!T9ZNHqN@ za*Gli*}TNh{R0=B-&>bPF4gfHi@E!f2`QQARgI{NlhoWuXcG?LpYQlIh_Y|BNHk0- z@|V;PnU2136X#(b)35Zm+|rJ0L|=Wk3AZ>`+Q&?$b(7htH6%)^bEUan%uUXk?^jSb zX#FT+NK<@`B~E?=X6q{}*P`4XZ*>|gBplz$HV9Z6kHfosQL+(9W*kqJ=4}+oY}@mz zFV80eFSj6Scz>E16eD9!HGIKHDlYV_b|^bLu(Q>@xEx3MS{uEjP&i#0e(8d?V!Am` z?{D{-U>{V;+BMFWKgBYd_OyTgSM)gNa4t5B#j7$qm%gKOU|LcYC$(HGjn&%vs%-F( zxjo~P?!r;~cB3iZDBwaA6Q271NT{$)z1su58xoiZu#xOIS(~3Raoe6gTdR{vs1loJHJpAGI zGy7P|HK`Y+x9Z#2rcA%T(Uk1&XHdHR0%~qlJ!ynJVOx1qxWF_1BKhhQls|{nBJ;?f z9d<6RdjsJFMyL3zBaI+j#f#sZ%$MKar^a*s))ndvPw}@ugQgDH;x}0nT;r|97_hPA zaQ}R}`4P-O{caDk8Eo2qa$#uP?nM8%4Fj+r1*E!BBITH>Rmc9a!fH5ey2wnpO9OMi zpVYrO1ZF=t_7MrMhKDM-T9y*@?pgig|=bPFuw%^2JHS7UaQOJ;TjRltT1moK2sqI zyn9>oNyt`nJ{P9t(UT%GF6~yX-?bZpsMrB({+g7t5VET?6NFwICMCG7Y^x^13r@vRMfJ3;+fi1M#e} z&x0i~_xG^^)hT<5j&aZtw~u$=38CyC?^?rVXA-xtmfu7h7HZDItMkt@OlQUEsn9cb@FqBo{{9P4C>H=1I(kwI(!ftQovxH-~*9 zZ&4K(`3-6ad#&@w1Z?+Rx6-Jl@J*!MB)vZ*p>y~UN8#N0ms>8Of2}NKU0r&^I;?lH zHSO4kB>;rN`8`b>SpqL2KK^bD9AsmyX@AS5w?XzxPxSzb2J^U0h0Ui8wUgYz75bl4 z-MPpANvLE03N}%G7w)F5&sA_(SYg_MLGulx2%IxRZ*N@pE{-6Eq4$L|uciJ>zryPCmp?7 z*ik_wRc~(8JAph zF7@N*>%6OV`F2}Jc@Va7a-9V1@?v+nXR1$QtV-1TCGG}21|CH~Hmk>`FW8W0>FoE1 zX3Q4DRYCU0c7b2*NpN$b(;z@|fy*2M8_p@sq{fCoZ$t053nMM7VT~!nr-ZXhDtYlo z>*zc-NdU6Z12oJ1Rc^Q0n`u)gYPiQ7i5)GplM2CyskZ`_tlReKCPx8crB{FJ_&Rq( zU~mok5FT}QCi5q5>NnmIE7;6Du@1>X&$k=z_a@199(%i=wg-Lh*~BX#rLlYScHf8Y z?8n978(tnei(Kn7)t}8OY&E5oJ(!bXtNxJ>a2xVO3Bbs}hBVUZL6br)_2b$y;$kEm zr*W3Ng+;qqMY)is8zUwAd;&#Z&U(#H*gsgw&96r7IQ~ytZ^6}O6SZvz2=3B0xVweo z?o!-}OMv3;?q004P~3}KaCd8Qcc)l!cldHY&sv|YH$Px*{bAh^yZ?pN@XpQN52ZKc0vUXghI+)=dPfemn^Jb%J zLq5F)J5Lx{+AW30AzdUTHzCVRV0}UM^=#1?4eSDFOp7-PMI9WY^A;7^z4iZx0rwig zZ|-bp&CGH#oEivDO0fUvf?zIH z0FVMs!oicSO*!4lGK#J<{ZX0tN8A(Xtxmb89_dh&*{MBO#^tyLT)veX#oS3tA@JSP z5sYYO7WKZFJKJS{DPM+9R$Y`f#kfikN5;YWDk5{&w_oRE01U!$G607X>LG~OAsaz! zZd`brdmD$sIPLWviY?tpR768MadTkBzsc!Sl{HQJmE-{oZXMCL*&oSzK{w{)0oz^;_(*y$^UqRKvHK}Q=tb@|z^rbNB9--=gPbywSEt-SxZ zvQnv~S*@|^yS}wN?u75d@cRwNO2Z%U%J9Mce`%COfkOy@mBDS8LeIgCZY{0{iAcgR z{Ut>I>!u2aB4n4cEjEkSwTjosR9ZcXkEip*-=4L~%L9(1)VSt#jn#(!DSRJPS zeb4)USfl@&FT$k45l2DcpbA3(0P~$KXuqhA?jxEg8!NGxn)wRB@a>2Gdf$-W|0FIV z=BIEk;*?=|MVHf7uS&q>kD{K(xbx9-Cht&e9EWe8V1hW{HC6F1$n&*+9 zF>J>mN~8Thsq!efNbuDA)W_SGvp#LKyKb)*_SSc}c7m%ef0&LmEZEzQCAP&I5`?t= z`>wQ4xpVxLGu}Cqo42*_pdXp3?}$4Dcc(usCnrHsxB7#&QQ%2<jQ%GLdz6#o#>ssYE$3hdPwaC^LTOXa~=y5`nOE$~G+(X3hu zh?9CpPL3DUEJ)Wqiun8AGIc&8{|&eAX^z5g0jx7}yWLz#WC14U9TbP6mZxjEs}EiG z(#P1<+Uv{B6YC3{Oo5san?Ss-LJ>o$8MQTdf`c~wPyIUN^z>(fQnr5XF&o3uEM$CpP1C#@4HckCj89GM3h6x*_v zDUbSt8{Ft?IahP-hGO#i@uKzC4~613l$@AZR#h8|>M#eOE9sfnkRX*^JpsYSYD<0l zjfJysmfbfXGXo-^&DNLP0~H>M#Q7i3RAt!PpwN6x<l(UPg@5c;xa6S zgZDG;n2v6rBM>buy!T{4aAjph9LLRNQ|a)$|E#Y>Ss1ow07Nbm6Wsh$X|e28pi*AO z?jXZ(bv@gv$7Xc$^6%jwg$ohHDV1{w;+cp_p#kT<^+m{Br~a`f4q&J3a*RX zo88774dM27WJMAGtwmg4duw~kd98r!?*uqJQ4}mdS}=7}+-^C;9tF_OQKh;5+seS> z$M=q=M{ka2hGsN>s;;`v)`_cQ{pJhG%gP_QXA&qaBi2a1OD5LS$|rSjwcDa1C{Wwf zY@^}Ki+=cKhcZdt_bO=}&W#KWU;(0hqpbXrp>qO)np<0AFygJ^eoE}~zG-hQDmMN+ z_j&GScvxF%)i)tfO3Tdy=FSjdrmoxRgM7Tv;By@~);atA+aH_#JZgQr%C%$pr9ZnwXb6jnn>I?*=up2vY70gk_tsm`< z+|)hyvOnVk8q$aOpPJrov9Xp&{9&C45(R#2G3H}ECAg5y8B^I@x3xXbxgC&&i+X}2 zQP7>SC-)-QeGd*3u>>e(G(wGi4hwHr+EFcvW6fEjsPw#tN#^4)byv&1-#bCroDYRipnj9 z@;Lx&uJkV}tC};ujmToxa#5|ADjZ@Na5y)w?T)s7vOn9erxlj$ZWq`0rWL|xtOqGD z?#V!u`eOI+*2|M$>Q={)W3WCU6YlWk5AnC?^`_Iv2bR&>+^pp)l7E0hgbd8`?N!) zjaq%8$;z>l=qYFVA)JEb9%EREJ@7CkptSEKwfOSNgDH*)6bhipCCHDYLvHRK*#9=3 z*N)ALY|>pmkC`eMCr?;Ng@M0|_j6;QVdAWK+BI04@_XijoCMsRLtbO}%YqT}nEn1k zg?gD*x%)_FGb+dTZ8+R8oURL)@#3iK^+@#L&(X%?JTvSwXX8FA;~saLIA?K^5x5jv z8i{<4n%lv@>?i-{Sn=XSC>y+yTo4i&U^>jhEgr%R| zzDhpM9jt|szaAvKmVC}ZRYp?xj-@Ce)8EHPr%oTBhvx+ZaW%tD>$F6x<2oQoKztDj zg@rDhN3)QETBIvh2QLM|iZ~_#s4ejD4RnC6!(zQzbj3rrzHC&yx?fc5ZUoC$e~~N|W?opY+w4lqP{ykaUWW?>IPRI~P-Vu_qptf7M^6ER1ypXKk zdFD9~3HojsL`PejO(7lX|&!+s3VcST+5J65lDJs6$bP@pg+4?`Dd<8V!J z!^H6-qnTUf!F-Ju3$R`h<8A+-=1p<<81EXP6Ink0W+vW-$Fh>A_mCMsc)b7B0|G@} zs+Mn0TXOimSC`nt9hwG1o8q1cTxPzBG{X^Yz8d}p{KTZKsR_wNqYs^Uk^I{kBKoB5 zwclWVWT~NdBwx7w)M)}bA>`8$X`1*fI{P`2watE&Ai60#SGHcBAlh{TRT&@wvfN|W z)hN#E=AYcGTqsKwjO5gY=t9CI-CM)pXr1rqPB)}Wqbl83eXyT=!F{G3B(ZQva%PtJ zAG@DvQ)!1`4!-(*??Uy`$d-H*VQAHl29%DQRr>AiO8QjL^*7&ugL62d9}pUbWY5D_ z7XfTe#QaB?r3MGdrS9Aoa%}Q2h$00$D*$(np|H@e2eJgnrR6d?Wl=F5xq($moa9XG^M=a~Jie7m{{6tKa^bBokUI!PNwI0N_m=|eAb z^gqwCf%Ymf+<@(K&`SVMD_|DDO^C5xD1qEumJbyQLJ@*VcP%B>cL+js8b6C+cP}2B zAIWO}i+RPyXl9GwNyrlypwO*BZ*6(`-|D!MQ9kbt~~ z)(-zGU-H)$913{b=TC&k1U!yrFVt3;mEu!@Kq^(qcq+4?D%*~qP({V85M$U(E+KwF z*Lj6oyxtx?M2v|{?#oRgTdz3uy5KkZZ|I#!vh4_b3$c*#9?)A?d*GkqX}!?APKIcd z*iZrt;-WZx$L(8poVgJs9Js{h^2oQjLSr~!WiE}~hZbEs$IXVW{!WI>htbb5h46;x zD)yjW=S(wJ68u;jZ0E7M^vLTQMAj`lg`kdek@-FH&GEjE=XML*6S?4 z`(eK3BvAw^t=e+L&rK4=?ms>9Z~#WT_1=F<-{3z$Nc2n`@tZb!D|w-J00S{bCyHe8 zO_M;<-?|mwYA&FIK03r@eKy8%_14mP_&cujgKjZ8^n5Z>pR}0bNM?1@wW--74NZBc zuK9Aap3JXC=`ERvGs-mo(lrO$paQr^Svcl4NqU1Zpmh|9`U&mhti7dus1s5g~L>8cEarK&uYr! zCR+^#i=h=hp`e@=vqhCwgnc2H$JveFQB+accWKFPDnISFCFIMAZCu1t2!~0dB*EmN%>c{8p7S0N20kZL%tBHturvF{OhtOH;oAWx=)A^&XltzdnM{NaGEHS z*Am_gqI+r}e6;KO`ij+#M~yPwFCI*fQiNpYG^0usjBO={xx4qzfxLMT&A#p5KxyQeYu z^?x2avZn|cL(&gloSa|62aJQ8SLyN*iPQf6r#cxx4Bu=MX~xphtrcL*Ir7&VjBb@V z_;+(F;4hFa6`BTZfi68iw;#1;jR=qt|M#q$UKloD3=iLXsmUF{pjaLXClU@L)l@vw zs)d#^(43T%Ou~|%#v0S`3{MJzG5UpSAbdcMbJu#@WDk>bj8cau<0S4!=ysht$F3{Y z5vqW+DD!EANngQ64wMp=EI&JvO{_mRiUyMVP8<{}Y^%qq)b4YV*hKx3A7&v1K;Sib zhe1u<+!G}R2ZzHZDi?pul&>!8+sFHI*%C-ehLB3z0Lo zHofN2$N)oPq>tE@I|#z`bNifRc~UH>U54(2pjmb9GQL{?4ynklr7k<8!~L;z{v|et zt|8y-Y3pKIf&FGFuShS8nohZtJpZ>L6E|1BeF>DxhWS=r>?BvR zG+8O(E@f0uhR%cM+y=5F$P(GH_*m(L081h3Bj8^8Pc#5TI5SLm3{g%}l#zN_?Jd|2 zIQM7wd;O;!Sh;hR&G6qlcRiCkh`dM!!J*+W57pC&0k)Ki@b3Lpy&$~BLVhD2WC33Q ze`GrkiF~`soDL*B0PQ!cp&L&hv>y+=?IyE3L$kGrAeNy+75rUYYEP%0T?I*|vHbC7 z>3grQo$~@11a}k{Q<$@QD~(z5g*FZ8XHQ!xFG8FceJJGE9i-zRjc_!7tj*ZqsBSLY zvt=p^ubW$ChA3KN$f^eMZZ>OJM$}aNz zVBV?wBI`wMWm4GmGgB^Szrx-p)fR(*t1;e&c?lFsDYg{qe7Pmx<_?oA34m16m_@Ss zfGI9=h)C}rq!}*HrHHNrLP7_Xj97$LSv1th!uS8o0Y0stD#ehEwh~4AWB70fIeXMN zH{VZwC`OtA$Gz!(B&ridB#v3Y95Qr2;GkVy6CiX1 z3II{o9S?u56ipb)$b`#yALhDpmcTn{HD$b;;}o!H_;0SU2K%Rsz{ekl<%d^YV9cO@ z4uUsgX#x1EKr~65j)--}SD%1By3KNa6q` zqTB>dyp2kxOhRRJ6EwjDN8li`0DPoREF)-r*Cu8gcgPv0vt$xcPNb?9T+UhzpElQcK5L%Q!?Hv8GaT-kshdN(tf3!~(i2g1SH4;vLw z8QC_G3I5<(LeE$(6a+vC!4{kHUut}ampM63E?K!4eJVqkT2jo191KYgKre-F`E4cD zUgUQp<&RJ@k(~H|omMsl=G(W>i$05f3ah(Clk0T>xJ9Hv&c{_U+UR@$ULo0$R9xQvAer3=By0WkFB?0h9nVNSI)#+d})@S)Sr33r3>lO zKYKU49y9%{c^q~qkBAX9=HoAal*ueo970VT%d0SR(w+*uSbW{`Ho}ps%=NDvme<1m zB8+6mkL3(?3Mo>gwgQ+_2-^rEebvYK9(VV}iG-k?5eq}`%lf_b3owYIC<$$bjSztY z+ye^ab0(tiA4x~z;{*jc>Xh}B7oyW$w>_flSj6U@|Qqz)xNrf}t zEZIhkEoY?jcMB33TY>AE=8Aplr%0PcA60k{)Zek=>A~9~dA&G>IhV55qo1~EYns3e_m!$#;Bicq=JN5JjS82sPXhxs+f) zkY#v@iHV6YA!9=A1_{>x>dYaRW>k`r|1@|Xiku-RC@2vn5@=L7@d1_Rfdop%=Nq$0x}@Z?Le zRy81+dMt@E6vdHm-6aMkx zfD*9Qkt$w{DZ8()?^)n^Ky#_Srn0}Psyxg!W8C4Rgi=#m>#{S5IrM34e0+3l4D;FG zqNn@8@4&sxf2ZoyCdZGgw1W4kmL%3W%#gF4K9M8AkucDW56P#TZN>g+9M5cU>!pj8 zZ(q-rF&U<@W-Ax>EtCd-u)-LLMC}R3AtugPs5bUAy!$t2i;j-|BfXvzL*#DNYtfI) z8gWR=fOKWs_^nciGM$W9ih$gBfKsA9SY`%k>)5=c5g;@%qemR!Y&Tkl%w@3*iZEI6 zxPJz1tgO&4+gfEbm7A^9S33y*A{J4vZXVz)R2S0aM2h59E{R`n4&lz-CquHGMh)q` zmnoP2Bt>9fjjLrJJB)#`z%tGEyPGhgob1i`n4xsBz{(Y(ZJ(Z9LO!BzlC<8MX7efx zr*A$QtIryI2K?P`i}dJ4l0|KgJ-B#z>C$8wjrGv6@zhbZ+*Av~15Wd7NGPVC+}Z#3_{YF{N!ph9H!S3IS!mXwv=4%9z0pVmuRK9yxQ?8glCkouySG>M5LW*M< z?|_wu*rHXG$x?IV(%7M21YHehh8mP`|aR2_VkUujnFo7^GBN zQ^8i5b^W*tdU>A}br^l&4H_mhb|e+MxGiBlz^J|l)30cYVo4HBoBbll3W8Za1nO&f zML_HQhl?hw4jlmG``JQ1#AG{En8ISV_PRgMo)8SUm%D&gm7f-_T!(jSlU(GE^oB@x z2KEzAJj=jT=s+kKBarY_UK2M30gg_G8l3lm1 zGKMHzpV2(MNS(Gl(-Q%E3KdeuGx^OxGUes*@OOFdpE?slpU9Dh zS1i@hC~KEk02|o9l3pAOI8IY(M258IMS+()xrdY%?{4YT=G*dbYT^R-Jp}*FB6)@n<wDDM~=+=JiSPAyL3nmrX zG%D2!KVutx8Eu*MM$j_&c@#ggNsiL%*8_F&0cl2;c{cFN+ZwXG@B1^iVB~A>?mp+7 zsr(&FX~xZP;qFDR@_n>~Tc2bh3M(bmN+atCQ?mS)ltaCztSBwe^!^JekLk#M>hNP^ z_w{}AKL9YaARz~MiU|Zum&Gp24h-LI=nKU3!rb7YoE4p(ck{+;mE%TSh8{c(ei!_s zqkV2$$bGT26?uDC;pv)s8$`tBNp82s4O1^f{~Z1=*0S~p z)J`tQRfi|4_hl{!P=qM!aMo$4ob_=`sQR5vyzmtFAJd3QZmeQO>N7q?1G}zq@O64Z zmNBL%Ntue6)YjMef=FJv(fUu&!}EW5cS4PzLziC`^eJLvaDy|!sg+wVZgq-7Hjg3t@V4Fl}!f`f8zMD`&V5uuj? zU&%k)6>t7{w7Lqta#g`Pb&-utZA**mC3>79B~rvBcBr?0Uns%DgZ^@%wg2x+k6PqA z60?A-1Zh#*>WDB@-T2vSsz&`66Gf;_p9T5KjhdK^!b#v6X{Ua}M?fz3DA%}1oE@CJ zB#Kk8-n(SHz40IpFw%IB_fJ=J00_j4!^rkayV4~B@x{ajbHz9Ot@$z1`0?=V&*zy& zP!6U%9eS3OS^lpiv$n7QTuEOv92*%6e;r$?$GAGA?={Cf{h~4w2f^RU*h}U#Z<>sS zPnDC4!Bw+4PU_92Bbot_sYkQLnPMn{tb?7cFjoc+uQoBZpr#nHs9>k}dqPfB;xsnN z|2_l~Dx(EQ-src9-mAQwjRe-*&g8uA(xa1qXfF`c)>a;?iR*$jcy?d^pEA0T9+x5a z^8ErhOr0%-Il0U_y?ub7ect`^^`cAiU?Y#lPw?uS&pf$$qCODy8BK-UowTfw+g}k~<0eq;qX);@7(nf5tOMqW2&K zd_ZG&?PegbIQ1W=|@Ur9RU!{ZfGNgW*WU7*T>>kmXb)o#ap{0a$F%y7*xcMPfbE zL)6jn_wWrPskWeau=FNkEH2J?doVhPZdfqYNW!70y*A*i{zhtWta!#j!xAc4e`)S> zyY!?o5%sQ@?~~@2M1d7`wpJZRa~&PrgoqJh*HM31eoG;*cQ&-l5NXIbr(juJ;Fnwy$|7+_9MghM$Z23zK;Z93{w8GoaM2> z#AWKNDNMBLiYTY8r{MNVj(6KVRI?j$Y*R}~)?P`|dMLP`KMT2u_L-NP?Ds7~1g_@` zZpz~tr31vw+@aCN83IPyyz;t^1cHv4S40W&0txkW;=+U61Fxmkj4F^phj zlX>dn0i{{_-($I21GDfJ6W!IO7H~^KOtilNo(>L$oD|E=b+LNyI9&m`4R600F}%!y z0_Ihs>6 zc3?X|E3nkaRC;Z&`8kSapZmLNcZdGs#)H)vn|a{t zmBQ_J#)_q-rGbHg0*;Nryp`XVJxJs$W)+erTO>PqC^C^l5qF%Uj@Cr+*fHMW*wJ7u zjq*7pQyu_inEi@UplYF}CZC-j>8+V_uOAtoOe@LZX6FfH8aAm0Nh>o;a~z?#2z+vV z9+Y=AU6{O);DJ?kYEDGn1RKJL;9-%kD-5F&pTO~Y?{R!K z1riw{rm?iM3SA-o-M@Gd-Ds`zd6^A&g)d6_h%wv9QIWW7Uz7}P_PkPtM?A@hBT<(h zsm&)|sVtQu0E##0*$Whtwv6_vk{4=LYT4c}^TxF#e4xvoC`h{bb_TRdutrVYadO5M zWpLBjkYSPusv4T-58aFUiHRFb3=kt)1z1wW5xQ5Yv>ePH zNI`T_;I1$OxZnguvI#%3vzM{g<-L>X2Sfeh{DqU-gD*6jYDOM{G_7%z{RUbRhVhM4 zD4|SQb`k_eDJD^1Pv{{&Z^aL?0mWV_u>YV;@0ANIvv674JNg#=yYXbwMD+T_OWJ}i zbwa8Dv)2l+D1FxwWCnjqeNr&*e8FBOS1Cr68O)eU7nMsB1~-kvQ`UxJBc~#+0HUiO z+#~8$Bfn40l64>|qyjj_bAIUNcj%0HCV0nP!a@9K7M>L?k zu8z#s#<6fAxtST67LtYBNev#9(`YfM(qvNN)@0n}E0v%mXH^vbH|t6KDT>H9Tbt#5 zhKO>Kda35W+0qs)g`rqq ztIaMH{?pv`9vAp7Itt9eZoZ^Qy>}@h zF-vvd@Zb|zX1Fnq?Wsk>SGj&+JBT4Q2Y&b%WG?v$)_JF-_s^#58%|3#?EZvz>)!TD zk4u^i0;zFv>afr_y`Vkb{*RPWm0w$2`CJk8 z;DA2$>M3BH`(|H!PIEp+Ae;w}DqBx2T#z#u2}BW>rPRznYTO&g%WzbfJe|D{yH09` ztMBee8E2|S>Am>VSsT@LYMFPz8A~Bc%1F3GUWZIrz{~OR==}Wc@UZ+2+MJ9NTko=g zI1XRDMyckcbLq8K5uO@Ta(Z^Qo_^bzcSF&>c`O`8u6n84Z)}j z;7v07^PG12oAqw;+Fl^&qroQ`&i#^^zL&iWP3_j@t$M0~!8TKiu-q6g!{HjwZqi9+VM4Hc|iwZI5r8Fz1BBJ? zr0Sv2$w?F0#HA{A#0dEpkbIy{HAnjo2^2@A*=|H+z`<1Gr&WF7W&$D&zvAvjCBGv^xX^VcL<=6Z3v;i8&4pf=H&f1Fe z6D=_w8;5Xka}L5_?q-eZt9K#)Lqq%?mq4Vzbb@gN<6+55(PR!kXLZhnzrm=hLiA0H zqG;^uo!5`^juveGd&`J-)&TiKn+77FctJ(Mljc{m=taIu6HvKEpt24u$)6>+8=&rFZ;~Pg&##c`l~pK>rng z;teR6158}H|K8?_c6q(3n(rNopB1?UT2bgKi@d(q+Y^{?kG;L(_ZrfA{l zsDc91viklON^)Dugq%x!OdCCUB9~U{0d3c8!_?QBPzKO#3Y)iRC%t1uEg(K77pX7o zP|Rl&z7f3(-X3y;fU`Br&yp#NL&A4dk^Qc&E6$Bf0I#P@G%< z;-9|t?y*{hFG-`68`8xU5!WQw{TcQ!;yS5kaK1S zHv#P1OH&!w}5 zi{3k@HvyZraLczIOUuleOhOZ_aFZzL{SL>aO&%OJ21Gst+*DMc7Zi;i(L&-8g3*YsM1iOqZX7<`+gz(t{xc`zF-_NO zYWq|iDDyw)$e=0ZpD#hEgRJ8ZJNZX7$}wpMG_@iw-{v=2dlO@W&0Ab=DpDpMo&?Y3 zV=3hDkrA^gQi)=Flq9WzBDs{@Ng32bl#C6Zo)y=1hnZu7ufMzF#zv-(^VhL3|M*kS z^pXex{m#re5ST^X4wjmijea!cRU(?Tx#8ZCGvvskx1_LSy7TX%p@qXx6xR}JxfMGW z0)x23Lhyc-3GnDHH!luDd7^HH&;hQZ6J929`q;#q zaQtXa&Ta)3DJeBsSqVLKSGjas+34iopf2w?Z^5<&YMBl>{HUnO#%>Zb@Jxgw<1QS4 ziv!64?C&8{jDF5tre)yBD6}{4C}b>%TdL1?j6KO! z!a)|2@l*cdpM+w&UgtC+@E<7R(DAV;Eg4^&f4QgjwPV~xZknid-Q1t@K(QC;XB)&jXB4TAxB3I4=Ght-oYH4;_btfeIDGYCQxx)}2YW z-k!5fXeT}VK+8o94{Am&hkyusFlB&fRvsR=ivr8C6N}qVMhCC-ftSTo`?oh-dN&(( zUD@YPWTX5T<9rDF{lu6hjv?ZT9O|mT%1;QJG%OA-P5`3*C?Pa1;vLcuGGrmT|GxVg za!fInNEurf+N5z?M1cE$Vl?(z9rn`$u4!I;*Igmwd(-DM(I?-+wv_}B`S9Pkq-72(ZYtD6Ij`AW5KnuhZi z_mk;BEU+UnQtpfJdz56-Ob#5}RJwolf88bNk?M*usHmi4Ihg5nB(u@Y4!oD9a*d?J zL0>qMtmq>H1ZvW*KApt`UIo6kO^CdewiMSs&0*4-ES75WXjh#+Hq;_I;L+$%Fsw@6 zK}PRC6O;o0k}hm^^kV_%3=%RJHi|#pTleO&l!Kiz#QBw0eA)zVkgM7Jd>$%Kf zPC7O{LI+NOV^TZ$CotaMClZWU9c#?f7G8bxQ#!=chf!{60^ zaY@ZQhe7X&RCF65Lg%shx`x$7dxC3cf<4hD+YcmEi8o$(Y^Ux`Sw0Zu@?<4p!Wl#D zp+-kPYtrM_t`lB=2L$7?h854|O)B2H>wXnM68-X#jc1tn-*wmXpdxgY@cKuTGwVvw zw;X%E>WRB+PJx$k8VR%zp-fzsE8pJ82s>zqVYQ3@<})6GG3$VE_M;|`8ljblNtpLE zY+KmT$?PL~Y47O3#Ax6%e?jA0W!Lkaqu*Osw*13cz)_Y;;H5&&1Np0Xk%#h<&CRK8 z@0S}UTUh~0^j^vnhFz<@PaLp0;&su;44*rZFllV%=K@$LYxl+JaWh~gV%n(hqQRbkn{ou$Gg zJbvFk&VQTh1_d%;g{(v;Fu*2f;lo+cSS0=(_6?@xL%Q zjTD^@r0!Dn-+WUPDPQ23%`%@!HQ9=qDyd$b!yy5(r3RTS7B0W8wYdxv=Z2$3#C_WS z?{~&`3(iIsi~3*5m26GP3liG1zxpADFyPAfZQ@q2>2<_d)W7TAHD=;@RS+9m8~?O% zIkUdJpA*G1ofCl8?>B** zZyykZoI`5*CXI$Obv)hkg5~+KR&wb1->qhHdyZGoD8)C z`v$`*GA=(i>`^~fnH6ko}>wjl0OxYrG z`K`2EqdwnvNA4g{6Wp6NgWh$ST-|x`SSR}YTCUXzP10?-dV5K4tc+QxdRg=Aa=$sW zv@=*42}o=%(TkWDP=y2Z@$?Q0sxyN0(6=Mq!XJv>x`#F|e;s=ZX-9QXZfw9g6(oz;R!%7+#}@jKrZzmiNxqW*|TdhYx~b8*K(v z%?`s{W4+g!0$FSXXlnL83+{#sq_a!-u{ijDENNgMqRw@rx*7C8O<6dV(nMEFCo(dM z&8S^YDvy)waYw#%z3iGM>c1rpqZT0y2i}Zt-K>W=i9NA3noRBo)h{U!CGb3mU4;IN z(j=pRaobeg$c%Sn^iF23iXs>mhpK=f*PkY(s`2~LV1;bhQn-%VQsT{emeL9V{^;MQ zY0S^^}nzRNFUk1qYVBO#hUXb$mfB% Wgjb= this.length){ - console.error("Memory.getItem: idx not in range."); - } - return this.data[(this.start + idx) % this.maxlen] - } - - /** - * Sample a batch - * @param batchSize (number) - * @return batch [] - */ - getBatch(batchSize){ - const arrLength = this.length; - const batch = { - 'obs0': [], - 'obs1': [], - 'rewards': [], - 'actions': [], - 'terminals': [], - }; - if (batchSize > this.length){ - return batch; - } - for (let b=0; b < batchSize; b++){ - let id = Math.floor(Math.random() * arrLength); - batch.obs0.push(this.obs0List[id]); - batch.obs1.push(this.obs1List[id]); - batch.rewards.push(this.rewardsList[id]); - batch.actions.push(this.actionsList[id]); - batch.terminals.push(this.terminals1List[id]); - } - return batch - } - - /** - * @param obs0 [] - * @param action (number) - * @param reward (number) - * @param obs1 [] - * @param terminal1 (boolean) - */ - append(obs0, action, reward, obs1, terminal1){ - if (this.length < this.maxlen){ - this.length += 1; - } - else if (this.length == this.maxlen) { - //this.obs0List[(this.start + this.length - 1) % this.maxlen].dispose(); - //this.obs1List[(this.start + this.length - 1) % this.maxlen].dispose(); - //this.actionsList[(this.start + this.length - 1) % this.maxlen].dispose(); - this.start = (this.start + 1) % this.maxlen; - } - else { - console.error("Memory.append: This should never be printed"); - } - this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0; - this.obs1List[(this.start + this.length - 1) % this.maxlen] = obs1; - this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward; - this.actionsList[(this.start + this.length - 1) % this.maxlen] = action; - this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1; - } -} \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/ddpg.js b/demo/webapp/public/js/ddpg/ddpg.js similarity index 74% rename from demo/webapp/public/js/DDPG/ddpg.js rename to demo/webapp/public/js/ddpg/ddpg.js index 63fd1cc..c55c0a2 100644 --- a/demo/webapp/public/js/DDPG/ddpg.js +++ b/demo/webapp/public/js/ddpg/ddpg.js @@ -29,13 +29,6 @@ class DDPG { let obsInput = tf.input({batchShape: [null, this.config.stateSize]}); let actionInput = tf.input({batchShape: [null, this.config.nbActions]}); - if (config.normalizeObservations){ - tf.layers.batchNormalization({ - scale: true, - center: true - }).apply(obsInput); - } - // Randomly Initialize actor network μ(s) this.actor.buildModel(obsInput); // Randomly Initialize critic network Q(s, a) @@ -56,7 +49,6 @@ class DDPG { return this.critic.predict(tfState, tfAct); }); }; - this.criticTargetWithActorTarget = (tfState) => { return tf.tidy(() => { const tfAct = this.actorTarget.predict(tfState); @@ -64,7 +56,6 @@ class DDPG { }); }; - this.actorOptimiser = tf.train.adam(this.config.actorLr); this.criticOptimiser = tf.train.adam(this.config.criticLr); @@ -72,23 +63,19 @@ class DDPG { for (let w = 0; w < this.critic.model.trainableWeights.length; w++){ this.criticWeights.push(this.critic.model.trainableWeights[w].val); } - this.actorWeights = []; for (let w = 0; w < this.actor.model.trainableWeights.length; w++){ this.actorWeights.push(this.actor.model.trainableWeights[w].val); } assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed); - - this.trainActorCt = 0; - this.trainCriticCt = 0; } /** * Distance Measure for DDPG * See parameter space noise Exploration paper - * obs (Tensor2d) Observations + * @param observations (Tensor2d) Observations */ distanceMeasure(observations) { return tf.tidy(() => { @@ -105,6 +92,11 @@ class DDPG { */ adaptParamNoise(){ const batch = this.memory.getBatch(this.config.batchSize); + if (batch.obs0.length == 0){ + assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed); + return [0]; + } + let distanceV = null; if (batch.obs0.length > 0){ @@ -123,17 +115,6 @@ class DDPG { return distanceV; } - /** - * Eval the actions and the Q function - * @param states (tf.tensor2d) - * @return actions and qValues - */ - eval(observation){ - const tfActions = this.perturbedActor.model.predict(observation); - const tfQValues = this.critic.model.predict([observation, tfActions]); - return {tfActions, tfQValues}; - } - /** * Get the estimation of the Q value given the state * and the action @@ -174,13 +155,13 @@ class DDPG { */ targetUpdate(){ // Define in js/DDPG/models.js - //assignModel(this.critic, this.criticTarget); - //assignModel(this.actor, this.actorTarget); targetUpdate(this.criticTarget, this.critic, this.config); targetUpdate(this.actorTarget, this.actor, this.config); } - trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals){ + trainCritic(batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals){ + + let costs; const criticLoss = this.criticOptimiser.minimize(() => { const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]); @@ -188,18 +169,18 @@ class DDPG { const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions1)); - return tf.sub(tfQTargets, tfQPredictions0).square().mean(); + const erros = tf.sub(tfQTargets, tfQPredictions0).square(); + costs = erros.buffer().values; + + return erros.mean(); }, true, this.criticWeights); + this.memory.appendBackWithCost(batch, costs); + const loss = criticLoss.buffer().values[0]; criticLoss.dispose(); - //if (this.trainCriticCt % 200 == 0 && this.trainCriticCt != 0){ targetUpdate(this.criticTarget, this.critic, this.config); - //console.log("Update"); - // Saniity Check - //} - this.trainCriticCt += 1; return loss; } @@ -215,14 +196,13 @@ class DDPG { const loss = actorLoss.buffer().values[0]; actorLoss.dispose(); - //sanityTfLoss.dispose(); return loss; } getTfBatch(){ // Get batch - const batch = this.memory.getBatch(this.config.batchSize); + const batch = this.memory.popBatch(this.config.batchSize); // Convert to tensors const tfActions = tf.tensor2d(batch.actions); const tfObs0 = tf.tensor2d(batch.obs0); @@ -237,12 +217,12 @@ class DDPG { _tfTerminals.dispose(); return { - tfActions, tfObs0, tfObs1, tfRewards, tfTerminals + batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals } } optimizeCritic(){ - const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); const loss = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); @@ -255,10 +235,10 @@ class DDPG { return loss; } - optimizeActor(it=1){ - const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + optimizeActor(){ + const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); - const loss = this.trainActor(tfObs0, it); + const loss = this.trainActor(tfObs0); tfActions.dispose(); tfObs0.dispose(); @@ -270,9 +250,9 @@ class DDPG { } optimizeCriticActor(){ - const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); - const lossC = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); + const lossC = this.trainCritic(batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); const lossA = this.trainActor(tfObs0); tfActions.dispose(); @@ -283,32 +263,4 @@ class DDPG { return {lossC, lossA}; } - - trainRecord(){ - - let lossValues = []; - - for (let i=0; i < 32; i++){ - - const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); - - const actorLoss = this.actorOptimiser.minimize(() => { - const tfAPredictions0 = this.actor.model.predict(tfObs0); - const loss = tfActions.sub(tfAPredictions0).square().mean(); - return loss; - }, true, this.actorWeights); - - const loss = actorLoss.buffer().values[0]; - lossValues.push(loss); - - actorLoss.dispose(); - tfActions.dispose(); - tfObs0.dispose(); - tfObs1.dispose(); - tfRewards.dispose(); - tfTerminals.dispose(); - } - console.log("Mean loss", mean(lossValues)); - } - } \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/ddpg_agent.js b/demo/webapp/public/js/ddpg/ddpg_agent.js similarity index 66% rename from demo/webapp/public/js/DDPG/ddpg_agent.js rename to demo/webapp/public/js/ddpg/ddpg_agent.js index 2a6050c..0906222 100644 --- a/demo/webapp/public/js/DDPG/ddpg_agent.js +++ b/demo/webapp/public/js/ddpg/ddpg_agent.js @@ -1,36 +1,45 @@ - // This class is called from js/DDPG/index.js class DDPGAgent { /** * @param env (metacar.env) Set in js/DDPG/index.js */ - constructor(env){ + constructor(env, config){ + this.stopTraining = false; this.env = env; + + config = config || {}; // Default Config this.config = { - "stateSize": 17, - "nbActions": 2, - "layerNorm": false, - "normalizeObservations": true, - "seed": 0, - "criticL2Reg": 0.01, - "batchSize": 64, - "actorLr": 0.0001, - "criticLr": 0.001, - "memorySize": 20000, - "gamma": 0.99, - "noiseDecay": 0.99, - "rewardScale": 1, - "nbEpochs": 500, - "nbEpochsCycle": 20, - "nbTrainSteps": 50, - "tau": 0.01, - "paramNoiseAdaptionInterval": 50, + "stateSize": config.stateSize || 17, + "nbActions": config.nbActions || 2, + "seed": config.seed || 0, + "batchSize": config.batchSize || 128, + "actorLr": config.actorLr || 0.0001, + "criticLr": config .criticLr || 0.001, + "memorySize": config.memorySize || 30000, + "gamma": config.gamme || 0.99, + "noiseDecay": config.noiseDecay || 0.99, + "rewardScale": config.rewardScale || 1, + "nbEpochs": config.nbEpochs || 200, + "nbEpochsCycle": config.nbEpochsCycle || 10, + "nbTrainSteps": config.nbTrainSteps || 110, + "tau": config.tau || 0.01, + "initialStddev": config.initialStddev || 0.1, + "desiredActionStddev": config.desiredActionStddev || 0.1, + "adoptionCoefficient": config.adoptionCoefficient || 1.01, + "actorFirstLayerSize": config.actorFirstLayerSize || 64, + "actorSecondLayerSize": config.actorSecondLayerSize || 32, + "criticFirstLayerSSize": config.criticFirstLayerSSize || 64, + "criticFirstLayerASize": config.criticFirstLayerASize || 64, + "criticSecondLayerSize": config.criticSecondLayerSize || 32, + "maxStep": config.maxStep || 800, + "stopOnRewardError": config.stopOnRewardError != undefined ? config.stopOnRewardError:true, + "resetEpisode": config.resetEpisode != undefined ? config.resetEpisode:false }; // From js/DDPG/noise.js - this.noise = new AdaptiveParamNoiseSpec(); + this.noise = new AdaptiveParamNoiseSpec(this.config); // Configure components. @@ -51,20 +60,20 @@ class DDPGAgent { this.ddpg = new DDPG(this.actor, this.critic, this.memory, this.noise, this.config); } - save(env){ + save(name){ /* Save the network */ - this.ddpg.critic.model.save('downloads://critic-model-ddpg-agent'); - this.ddpg.actor.model.save('downloads://actor-model-ddpg-agent'); + this.ddpg.critic.model.save('downloads://critic-' + name); + this.ddpg.actor.model.save('downloads://actor-'+ name); } - async restore(){ + async restore(folder, name){ /* Restore the weights of the network */ - this.ddpg.critic.model = await tf.loadModel('http://localhost:3000/public/models/ddpg/critic-model-ddpg-agent.json'); - this.ddpg.actor.model = await tf.loadModel("http://localhost:3000/public/models/ddpg/actor-model-ddpg-agent.json"); + this.ddpg.critic.model = await tf.loadModel('http://localhost:3000/public/models/'+folder+'/critic-'+name+'.json'); + this.ddpg.actor.model = await tf.loadModel("http://localhost:3000/public/models/"+folder+"/actor-"+name+".json"); } /** @@ -103,11 +112,6 @@ class DDPGAgent { stepTrain(tfPreviousStep, mPreviousStep){ // Get actions const tfActions = this.ddpg.perturbedPrediction(tfPreviousStep); - //const TruetfActions = this.ddpg.perturbedPrediction(tfPreviousStep); - //const rdNormal = tf.randomNormal(TruetfActions.shape, 0, this.noisyActions, "float32", this.config.seed); - //const noisyActions = TruetfActions.add(rdNormal); - //const tfActions = tf.clipByValue(noisyActions, -1, 1); - // Step in the environment with theses actions let mAcions = tfActions.buffer().values; let mReward = this.env.step([mAcions[0], mAcions[1]]); @@ -116,43 +120,31 @@ class DDPGAgent { let mState = this.env.getState().linear; let tfState = tf.tensor2d([mState]); let mDone = 0; - - if (mReward == -1){ + if (mReward == -1 && this.config.stopOnRewardError){ mDone = 1; } - // Add the new tuple to the buffer this.ddpg.memory.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); - - // Dispose tensor + // Dispose tensors tfPreviousStep.dispose(); - //TruetfActions.dispose(); - //noisyActions.dispose(); tfActions.dispose(); - //rdNormal.dispose(); - //rd.dispose(); - //trueTfActions.dispose(); - return {mDone, mState, tfState} - } - initTrainParam(){ - this.stopTraining = false; - this.noisyActions = 2.0; + return {mDone, mState, tfState}; } /** * Train DDPG Agent */ async train(realTime){ - this.initTrainParam(); + this.stopTraining = false; // One epoch for (let e=0; e < this.config.nbEpochs; e++){ // Perform cycles. this.rewardsList = []; this.stepList = []; this.distanceList = []; + document.getElementById("trainingProgress").innerHTML = "Progression: "+e+"/"+this.config.nbEpochs+"
"; for (let c=0; c < this.config.nbEpochsCycle; c++){ - if (c%10==0){ logTfMemory(); } @@ -164,11 +156,11 @@ class DDPGAgent { let step = 0; console.time("LoopTime"); - for (step=0; step < 800; step++){ + for (step=0; step < this.config.maxStep; step++){ let rel = this.stepTrain(tfPreviousStep, mPreviousStep); mPreviousStep = rel.mState; tfPreviousStep = rel.tfState; - if (rel.mDone){ + if (rel.mDone && this.config.stopOnRewardError){ break; } if (this.stopTraining){ @@ -183,6 +175,9 @@ class DDPGAgent { let distance = this.ddpg.adaptParamNoise(); this.distanceList.push(distance[0]); + if (this.config.resetEpisode){ + this.env.reset(); + } this.env.randomRoadPosition(); tfPreviousStep.dispose(); console.log("e="+ e +", c="+c); @@ -190,28 +185,25 @@ class DDPGAgent { //this.ddpg.targetUpdate(); await tf.nextFrame(); } - if (this.ddpg.memory.length == this.config.memorySize){ - this.noisyActions = Math.max(0.1, this.noisyActions * this.config.noiseDecay); + if (e > 5){ this.ddpg.noise.desiredActionStddev = Math.max(0.1, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev); let lossValuesCritic = []; let lossValuesActor = []; - console.time("Training"); - for (let t=0; t < 100; t++){ - let lossC = this.ddpg.optimizeCritic(); + console.time("Training"); + for (let t=0; t < this.config.nbTrainSteps; t++){ + let {lossC, lossA} = this.ddpg.optimizeCriticActor(); lossValuesCritic.push(lossC); - } - for (let t=0; t < 100; t++){ - let lossA = this.ddpg.optimizeActor(); lossValuesActor.push(lossA); } console.timeEnd("Training"); console.log("desiredActionStddev:", this.ddpg.noise.desiredActionStddev); setMetric("CriticLoss", mean(lossValuesCritic)); setMetric("ActorLoss", mean(lossValuesActor)); - } + } + setMetric("Reward", mean(this.rewardsList)); setMetric("EpisodeDuration", mean(this.stepList)); - setMetric("Distance", mean(this.distanceList)); + setMetric("NoiseDistance", mean(this.distanceList)); await tf.nextFrame(); } diff --git a/demo/webapp/public/js/DDPG/index.js b/demo/webapp/public/js/ddpg/index.js similarity index 56% rename from demo/webapp/public/js/DDPG/index.js rename to demo/webapp/public/js/ddpg/index.js index 2eacb40..07e5927 100644 --- a/demo/webapp/public/js/DDPG/index.js +++ b/demo/webapp/public/js/ddpg/index.js @@ -2,58 +2,47 @@ let levelUrl = metacar.level.level2; var env = new metacar.env("canvas", levelUrl); -env.setAgentMotion(metacar.motion.ControlMotion, {}); -env.setAgentLidar({pts: 4, width: 2, height: 9, pos: 1}) - -RECORD = false; +env.setAgentMotion(metacar.motion.ControlMotion, {}); +env.setAgentLidar({pts: 5, width: 3, height: 7, pos: -0.5}) // js/DDPG/ddpg.js -var agent = new DDPGAgent(env); +var agent = new DDPGAgent(env, { + stateSize: 26, + resetEpisode: true +}); -initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "Distance"]); +initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "NoiseDistance"]); env.loop(() => { let state = env.getState(); - - if (RECORD){ - agent.ddpg.memory.append(state.linear, [state.a, state.steering], 1, [1, 1, 1], 1); - } - displayState("realtime_viewer", state.lidar, 200, 200); let reward = env.getLastReward(); const qValue = agent.getQvalue(state.linear, [state.a, state.steering]); displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]); }); - env.load().then(() => { - - // Train agent - env.addEvent("train", () => { - agent.train(false); - }); - - env.addEvent("play", () => { - agent.play(); + env.addEvent("train [Background]", () => { + let train = confirm("The training process takes some time and might slow this tab. Do you want to continue? \n You can also load a pre-trained model."); + if (train){ + env.render(false); + agent.train(false); + } }); - env.addEvent("record", () => { - RECORD = true; + env.addEvent("Train [Show the training]", () => { + env.steping(false); + agent.train(true); }); - env.addEvent("randomPos", () => { + env.addEvent("shuffle", () => { env.randomRoadPosition(); - }); - - env.addEvent("stopRecord", () => { - RECORD = false; - }); - - env.addEvent("TrainRealTime", () => { - env.steping(false); - agent.train(true); + }) + + env.addEvent("play", () => { + agent.play(); }); env.addEvent("stop", () => { @@ -63,11 +52,11 @@ env.load().then(() => { env.addEvent("reset_env"); env.addEvent("save", () => { - agent.save(); + agent.save("model-ddpg-road"); }); env.addEvent("load", () => { - agent.restore() + agent.restore("ddpg-road", "model-ddpg-road") }); }); diff --git a/demo/webapp/public/js/ddpg/memory.js b/demo/webapp/public/js/ddpg/memory.js new file mode 100644 index 0000000..b4921fb --- /dev/null +++ b/demo/webapp/public/js/ddpg/memory.js @@ -0,0 +1,240 @@ + +class Memory { + + /** + * @param maxlen (number) Buffer limit + */ + constructor(maxlen){ + this.maxlen = maxlen; + this.buffer = []; + this.priorBuffer = []; + } + + /** + * Sample a batch + * @param batchSize (number) + * @return batch [] + */ + getBatch(batchSize){ + const batch = { + 'obs0': [], + 'obs1': [], + 'rewards': [], + 'actions': [], + 'terminals': [], + }; + + if (batchSize > this.priorBuffer.length){ + console.warn("The size of the replay buffer is < to the batchSize. Return empty batch."); + return batch; + } + + for (let b=0; b < batchSize/2; b++){ + let id = Math.floor(Math.random() * this.priorBuffer.length); + batch.obs0.push(this.priorBuffer[id].obs0); + batch.obs1.push(this.priorBuffer[id].obs1); + batch.rewards.push(this.priorBuffer[id].reward); + batch.actions.push(this.priorBuffer[id].action); + batch.terminals.push(this.priorBuffer[id].terminal); + } + return batch + } + + _bufferBatch(batchSize){ + const batch = { + 'obs0': [], + 'obs1': [], + 'rewards': [], + 'actions': [], + 'terminals': [], + }; + + for (let b=0; b < batchSize/2; b++){ + let nElem = this.buffer.pop(); + batch.obs0.push(nElem.obs0); + batch.obs1.push(nElem.obs1); + batch.rewards.push(nElem.reward); + batch.actions.push(nElem.action); + batch.terminals.push(nElem.terminal); + } + + for (let b=0; b < batchSize/2; b++){ + let id = Math.floor(Math.random() * this.buffer.length); + batch.obs0.push(this.buffer[id].obs0); + batch.obs1.push(this.buffer[id].obs1); + batch.rewards.push(this.buffer[id].reward); + batch.actions.push(this.buffer[id].action); + batch.terminals.push(this.buffer[id].terminal); + this.buffer.splice(id, 1); + } + + return batch + } + + _addRandomBufferBatch(batchSize, batch){ + for (let b=0; b < batchSize; b++){ + let id = Math.floor(Math.random() * this.buffer.length); + batch.obs0.push(this.buffer[id].obs0); + batch.obs1.push(this.buffer[id].obs1); + batch.rewards.push(this.buffer[id].reward); + batch.actions.push(this.buffer[id].action); + batch.terminals.push(this.buffer[id].terminal); + this.buffer.splice(id, 1); + } + return batch + } + + /** + * Sample a batch + * @param batchSize (number) + * @return batch [] + */ + popBatch(batchSize){ + let originalBatchSize = batchSize; + let priorBufferBatchSize; + let bufferBatchSize; + if (batchSize % 2 != 0){ + console.warn("Batch size should be a even.") + } + if (this.priorBuffer.length < batchSize/2){ + //console.log("get full batch from buffer"); + const batch = this._bufferBatch(batchSize); + console.assert(batch.obs0.length == batchSize); + return batch; + } + const batch = { + 'obs0': [], + 'obs1': [], + 'rewards': [], + 'actions': [], + 'terminals': [], + }; + if (batchSize > this.length){ + console.warn("The size of the replay buffer is < to the batchSize. Return empty batch."); + return batch; + } + + if (this.buffer.length > 0){ + //console.log("Get half of prior and other from buffer."); + batchSize = batchSize / 2; + } + else{ + //console.log("Get all from priorBuffer"); + } + + for (let b=0; b < batchSize; b++){ + let id = Math.floor(Math.random() * this.priorBuffer.length); + batch.obs0.push(this.priorBuffer[id].obs0); + batch.obs1.push(this.priorBuffer[id].obs1); + batch.rewards.push(this.priorBuffer[id].reward); + batch.actions.push(this.priorBuffer[id].action); + batch.terminals.push(this.priorBuffer[id].terminal); + this.priorBuffer.splice(id, 1); + } + + if (this.buffer.length > 0){ + this._addRandomBufferBatch(batchSize, batch); + } + console.assert(batch.obs0.length == originalBatchSize); + return batch + } + + _insert(element, array) { + if (array.length == 0 || element.cost < array[0].cost || array[0].cost == null){ + array.unshift(element); + return array; + } + array.splice(this._locationOf(element, array) + 1, 0, element); + return array; + } + + _locationOf(element, array, start, end) { + start = start || 0; + end = end || array.length; + + var pivot = parseInt(start + (end - start) / 2, 10); + + if (end-start <= 1 || array[pivot] === element) return pivot; + + if (array[pivot].cost != null && array[pivot].cost < element.cost) { + return this._locationOf(element, array, pivot, end); + } else { + return this._locationOf(element, array, start, pivot); + } + } + + /** + * @param batch (Object) from getBatch() + * @param cost (number) Cost associated with each row of the batch + */ + appendBackWithCost(batch, costs){ + for (let b=0; b < batch.obs0.length; b++){ + if (this.buffer.length == this.maxlen){ + this.buffer.shift(); + } + this._insert({ + obs0: batch.obs0[b], + action: batch.actions[b], + reward: batch.rewards[b], + obs1: batch.obs1[b], + terminal: batch.terminals[b], + cost: costs[b] + }, this.buffer); + } + console.assert(this.buffer.length <= this.maxlen); + } + + /** + * @param obs0 [] + * @param action (number) + * @param reward (number) + * @param obs1 [] + * @param terminal1 (boolean) + */ + append(obs0, action, reward, obs1, terminal){ + if (this.priorBuffer.length == this.maxlen){ + this.priorBuffer.shift(); + } + this.priorBuffer.push({ + obs0: obs0, + action: action, + reward: reward, + obs1: obs1, + terminal: terminal, + cost: null + }); + console.assert(this.priorBuffer.length <= this.maxlen); + } +} +/* +var mem = new Memory(20000); +Math.seedrandom(0); +console.assert(mem.length == 0); + +var array = []; +for (let i=1; i < 40000; i++){ + mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i); +} + +console.assert(mem.length == 20000); +console.assert(mem.list[0].obs0 == "obs0-20000"); +console.assert(mem.list[19999].obs0 == "obs0-39999"); + +let batch = mem.getBatch(32); + +console.assert(batch.obs0.length == 32); +console.assert(mem.length == 20000 - 32); + +let costs = []; +for (i=31; i >= 0; i--){ + costs.push(i); +} +mem.appendBackWithCost(batch, costs); + +console.log(mem.list); + +/* +for (let i=1; i < 64; i++){ + mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i); +} +*/ \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/models.js b/demo/webapp/public/js/ddpg/models.js similarity index 80% rename from demo/webapp/public/js/DDPG/models.js rename to demo/webapp/public/js/ddpg/models.js index 65c0edb..7823acc 100644 --- a/demo/webapp/public/js/DDPG/models.js +++ b/demo/webapp/public/js/ddpg/models.js @@ -18,45 +18,29 @@ function copyModel(model, instance){ } /** - * Usefull method to copy a model - * @param model Actor - * @param perturbedActor Actor + * Copy the value of of the model into the perturbedModel and + * add a random pertubation + * @param model Actor|Critic instance + * @param perturbedActor Actor|Critic instance * @param stddev (number) * @return Copy of the model */ -function assignAndStd(actor, perturbedActor, stddev, seed){ +function assignAndStd(model, perturbedModel, stddev, seed){ return tf.tidy(() => { - const weights = actor.model.trainableWeights; + const weights = model.model.trainableWeights; for (let m=0; m < weights.length; m++){ - let shape = perturbedActor.model.trainableWeights[m].val.shape; + let shape = perturbedModel.model.trainableWeights[m].val.shape; let randomTensor = tf.randomNormal(shape, 0, stddev, "float32", seed); let nValue = weights[m].val.add(randomTensor); - perturbedActor.model.trainableWeights[m].val.assign(nValue); + perturbedModel.model.trainableWeights[m].val.assign(nValue); } }); } /** - * Usefull method to copy a model - * @param model Actor - * @param perturbedActor Actor - * @return Copy of the model - */ -function assignModel(model, targetModel){ - return tf.tidy(() => { - const weights = model.model.trainableWeights; - for (let m=0; m < weights.length; m++){ - let nValue = weights[m].val; - targetModel.model.trainableWeights[m].val.assign(nValue); - } - }); -} - - -/** - * Usefull method to copy a model - * @param target Actor|Critic - * @param perturbedActor Actor|Critic + * Update the target models + * @param target Actor|Critic instance + * @param perturbedActor Actor|Critic instance * @param config (Object) * @return Copy of the model */ @@ -90,6 +74,10 @@ class Actor{ this.stateSize = config.stateSize; this.nbActions = config.nbActions; this.layerNorm = config.layerNorm; + + this.firstLayerSize = config.actorFirstLayerSize; + this.secondLayerSize = config.actorSecondLayerSize; + this.seed = config.seed; this.config = config; this.obs = null; @@ -102,24 +90,22 @@ class Actor{ buildModel(obs){ this.obs = obs; - // First layer with BatchNormalization + // First layer this.firstLayer = tf.layers.dense({ - units: 64, + units: this.firstLayerSize, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), - activation: 'relu', // relu is add later + activation: 'relu', useBias: true, biasInitializer: "zeros" }); - - // Second layer with BatchNormalization + // Second layer this.secondLayer = tf.layers.dense({ - units: 32, + units: this.secondLayerSize, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), - activation: 'relu', // relu is add later + activation: 'relu', useBias: true, biasInitializer: "zeros" }); - // Ouput layer this.outputLayer = tf.layers.dense({ units: this.nbActions, @@ -129,12 +115,13 @@ class Actor{ useBias: true, biasInitializer: "zeros" }); - + // Actor prediction this.predict = (tfState) => { return tf.tidy(() => { if (tfState){ obs = tfState; } + let l1 = this.firstLayer.apply(obs); let l2 = this.secondLayer.apply(l1); @@ -155,6 +142,11 @@ class Critic { this.stateSize = config.stateSize; this.nbActions = config.nbActions; this.layerNorm = config.layerNorm; + + this.firstLayerSSize = config.criticFirstLayerSSize + this.firstLayerASize = config.criticFirstLayerASize; + this.secondLayerSize = config.criticSecondLayerSize; + this.seed = config.seed; this.config = config; this.obs = null; @@ -170,30 +162,28 @@ class Critic { this.obs = obs; this.action = action; + // Used to merged the two first Layer later. this.add = tf.layers.add(); - // First layer with BatchNormalization + // First layer this.firstLayerS = tf.layers.dense({ - units: 64, + units: this.firstLayerSSize, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later useBias: true, biasInitializer: "zeros" }); - - // First layer with BatchNormalization + // First layer this.firstLayerA = tf.layers.dense({ - units: 64, + units: this.firstLayerASize, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later useBias: true, biasInitializer: "zeros" }); - - // Second layer with BatchNormalization + // Second layer this.secondLayer = tf.layers.dense({ - //inputShape: [this.config.batchSize, 64 + this.nbActions], // Previous layer + action - units: 32, + units: this.secondLayerSize, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'relu', useBias: true, @@ -210,7 +200,7 @@ class Critic { biasInitializer: "zeros" }); - // Actor prediction + // Critic prediction this.predict = (tfState, tfActions) => { return tf.tidy(() => { if (tfState && tfActions){ @@ -220,7 +210,7 @@ class Critic { let l1A = this.firstLayerA.apply(action); let l1S = this.firstLayerS.apply(obs) - + // Merged layers let concat = this.add.apply([l1A, l1S]) let l2 = this.secondLayer.apply(concat); @@ -228,6 +218,7 @@ class Critic { return this.outputLayer.apply(l2); }); } + const output = this.predict(); this.model = tf.model({inputs: [obs, action], outputs: output}); } diff --git a/demo/webapp/public/js/DDPG/noise.js b/demo/webapp/public/js/ddpg/noise.js similarity index 100% rename from demo/webapp/public/js/DDPG/noise.js rename to demo/webapp/public/js/ddpg/noise.js diff --git a/demo/webapp/public/js/level1.js b/demo/webapp/public/js/policy_monte_carlo/index.js similarity index 96% rename from demo/webapp/public/js/level1.js rename to demo/webapp/public/js/policy_monte_carlo/index.js index c40dc16..b8b5632 100644 --- a/demo/webapp/public/js/level1.js +++ b/demo/webapp/public/js/policy_monte_carlo/index.js @@ -7,7 +7,7 @@ var env = new metacar.env("canvas", levelUrl); var agent = new PolicyAgent(env); env.loop(() => { - let state = env.getState(); + let state = env.getState().lidar; displayState("realtime_viewer", state, 200, 200); let scores = agent.getStateValues(state); let reward = env.getLastReward(); diff --git a/demo/webapp/public/js/policy_agent.js b/demo/webapp/public/js/policy_monte_carlo/policy_agent.js similarity index 96% rename from demo/webapp/public/js/policy_agent.js rename to demo/webapp/public/js/policy_monte_carlo/policy_agent.js index 08cc158..b8c8bb6 100644 --- a/demo/webapp/public/js/policy_agent.js +++ b/demo/webapp/public/js/policy_monte_carlo/policy_agent.js @@ -179,14 +179,12 @@ class PolicyAgent { */ this.valueModel = await tf.loadModel('https://metacar-project.com/public/models/policy/value-model-policy-agent.json'); this.policyModel = await tf.loadModel("https://metacar-project.com/public/models/policy/policy-model-policy-agent.json"); - //this.valueModel = await tf.loadModel('http://localhost:3000/public/models/policy/value-model-policy-agent.json'); - //this.policyModel = await tf.loadModel("http://localhost:3000/public/models/policy/policy-model-policy-agent.json"); } play(){ tf.tidy(() => { // Get the current state - const st = tf.tensor2d(this.env.getState(), [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]); + const st = tf.tensor2d(this.env.getState().lidar, [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]); // Predict the policy const softmax = this.policyModel.predict(st); // Get the action @@ -231,7 +229,8 @@ class PolicyAgent { console.time("Exploring"); for (var step = 0; step < this.nb_step; step++) { // Get the current state - const array_st = this.env.getState(true); + let array_st = this.env.getState().linear; + array_st = array_st.slice(0, array_st.length - 1); // Convert the state into a tensor //const st = tf.tensor(array_st, [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]); const st = tf.tensor2d([array_st]); diff --git a/demo/webapp/public/js/level0.js b/demo/webapp/public/js/q_table/index.js similarity index 92% rename from demo/webapp/public/js/level0.js rename to demo/webapp/public/js/q_table/index.js index 4e44e77..4e6f42b 100644 --- a/demo/webapp/public/js/level0.js +++ b/demo/webapp/public/js/q_table/index.js @@ -12,8 +12,8 @@ var agent = new QTableAgent(env, 2); env.loop(() => { let state = env.getState(); - displayState("realtime_viewer", state, 200, 200); - let scores = agent.getStateValues(state); + displayState("realtime_viewer", state.lidar, 200, 200); + let scores = agent.getStateValues(state.lidar); let reward = env.getLastReward(); displayScores("realtime_viewer", scores, reward, ["Top", "Left", "Right"]); }); diff --git a/demo/webapp/public/js/q_table_agent.js b/demo/webapp/public/js/q_table/q_table_agent.js similarity index 96% rename from demo/webapp/public/js/q_table_agent.js rename to demo/webapp/public/js/q_table/q_table_agent.js index 9a96e22..e3ebc8d 100644 --- a/demo/webapp/public/js/q_table_agent.js +++ b/demo/webapp/public/js/q_table/q_table_agent.js @@ -54,7 +54,7 @@ class QTableAgent { play(){ // Get the current state - let state = this.env.getState(); + let state = this.env.getState().lidar; state = state.toString(); // In this state in not in the Q(s, a) function if (!(state in this.Q)){ @@ -108,7 +108,7 @@ class QTableAgent { console.log("episode=", ep, "eps=", eps, "mean_reward", mean(mean_reward)); } mean_reward = []; - let st = this.env.getState().toString(); + let st = this.env.getState().lidar.toString(); let act; let gamma = 0.99; let st2; @@ -117,7 +117,7 @@ class QTableAgent { act = this.pickAction(st, eps); let reward = this.env.step(act); mean_reward.push(reward); - st2 = this.env.getState().toString(); + st2 = this.env.getState().lidar.toString(); // Pick greedy action (eps = 0) act2 = this.pickAction(st2, 0.); this.createStateIfNotExist(st2); diff --git a/demo/webapp/public/js/traffic/index.js b/demo/webapp/public/js/traffic/index.js new file mode 100644 index 0000000..58531dd --- /dev/null +++ b/demo/webapp/public/js/traffic/index.js @@ -0,0 +1,67 @@ + +let levelUrl = metacar.level.level3; + +var env = new metacar.env("canvas", levelUrl); + +env.setAgentMotion(metacar.motion.ControlMotion, {}); +env.setAgentLidar({pts: 7, width: 3, height: 7, pos: -0.5}) + +// js/DDPG/ddpg.js +var agent = new DDPGAgent(env, { + stateSize: 50, + desiredActionStddev: 0.3, + initialStddev: 0.3 +}); + +initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "NoiseDistance"]); + +env.loop(() => { + let state = env.getState(); + displayState("realtime_viewer", state.lidar, 200, 200); + let reward = env.getLastReward(); + const qValue = agent.getQvalue(state.linear, [state.a, state.steering]); + displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]); +}); + +env.load().then(() => { + + env.addEvent("train [Background]", () => { + let train = confirm("The training process takes some time and might slow this tab. Do you want to continue? \n You can also load a pre-trained model."); + if (train){ + env.render(false); + agent.train(false); + } + }); + + env.addEvent("Train [Show the training]", () => { + env.steping(false); + agent.train(true); + }); + + env.addEvent("shuffle", () => { + env.randomRoadPosition(); + }) + + env.addEvent("play", () => { + agent.play(); + }); + + env.addEvent("stop", () => { + agent.stop(); + }); + + env.addEvent("reset_env"); + + env.addEvent("save", () => { + agent.save("model-ddpg-traffic"); + }); + + env.addEvent("load", () => { + agent.restore("ddpg-traffic", "model-ddpg-traffic") + }); +}); + + + + + diff --git a/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json b/demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.json similarity index 79% rename from demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json rename to demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.json index 3cc61fd..f5fb493 100644 --- a/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json +++ b/demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.json @@ -1 +1 @@ -{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,17],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-agent.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[17,64],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[32,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]} \ No newline at end of file +{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,26],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-road.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[26,64],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[32,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.weights.bin b/demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.weights.bin new file mode 100644 index 0000000000000000000000000000000000000000..8b9076defef7d22ca0efa7ce4ea75ef8a75ab6bf GIT binary patch literal 15496 zcmW-ohd-D9_r}daRDBrnk1i8pk(q;hZe#%~b-{GzzKUukFdZ z>1YN67Sd)%9lE5yzg>;dn&u09APg)2j=5=x}BiCp-5(a}Ci&`}|@U zlCq+JHht>TbXR4zvHOS++$Sa&^A!8pCJ=pi_D`-PkA6LG1FJJIA zpEj1du(7g+*c7&fb~tT^OKQn5%PdGvU(5XzVG zoQp>rcT{HvtO%-vQt|QdCoPNS&B-RO^W5HAuFSbWS0GEndGzS)GhQ>L z3aq~MGO67M*xJ$BSa&>-X0DLMxS9^zgG>Z*xQe%C<$BeWs*{7)7!aXJdG`;YdtTcYx7f9A3( zf~2*lQesUDKiA2Kd>6Ud)=xOe?Uo#1En_Tju3QrSk+z_?7x_@X?l$}RH-SRjZ9#Hm zIO`LhgzA+&u>X)hIf|uIX!b&ywJ-ojZ#)DU?!Caby&}hDor2)~@7Vh1O=$AzD9&C! z8#NQ}Q;~HJJy{!$JH0J2*>kz=i@+%OaP1+yvu@-4stf2rd^=UxSaDO*N@(V36>`~o zh6!<(SXSdt8a!0Re5EInq1zUcF<68e$&ct(^gA}Np@&)|ooVID{j|_MAGi1TaZkR^ zg%WcSGTil;BwrL^;GKJ*cOaN@HAdrvlAaS)fTC;F;qOd2x%)Opqx zpQ>-7(6O29>^Vbj$C?u?^N^oF5M3$rT+m-C(%Vu6?)HJ;7)B;tBrSx;vR(PIKhMDSR z;NE^5bkYi7Ny|rOW@}3)x}2!*s3S#%>CuB*HLOfTjSfzmgKg>t%-~TJ{E-YrLvLGp znI=nzgg!9q_^&XbW{B^a1=OCRNpU;uadck_ipLQ)-)=)Y#LzpRP|JJy2nooc$& zsteM!MN~3wG7bdk;B4VfT*sDNGM%%Txxo$gLZ*S%U#O=4 zFo*0Nj8;1-o)$ zbN9pfF~*c6c8VPc+{v1;9L;-30&{!xAs?izm(TAxR8n|Cau z13H_zv&+{(BR7HU&wOI@zn^9*F|Cl8u!FYwegMrk8Sqf!D;%m#VCyz4q9)mV!lFv5 zTM|gK-5dCXZOPy|ZVVb)3$fpqg-|k62S=~&f$dGJDWoJHY*jqibv;c+AKY+gQX>6s zdB9!dG)TO^8*<8uNHX*}%+`#c@g~>V#I@z@_B2oaf>0uieEOKR{c2*z{%e9$PzG-m zT;Rm?M4YU$gEt<1g7~+Nl-{NZ2KSO#)rMFBtn6n6BCgoCy^0-4xXQg-bCxWE){x_? zSDgBiyHN5-*kn1)O%({E6q z7nwqEJJO%cD%7NdVs4b2;{)+`53oa8#Cqo^q4a$}8gO;Na-RyL0_A(yiT7@8%IkC#T^mL)Fq=F4cx@lBwue=fnIPJ60&R6|;4>uJy1izMA2 zz-#{?kY06)tDSU;i?b?ZF^UIq#+=DCw|za^>fDZtM()PI*d+R%=|tCik}=h%f&Vu1 z4DAevCOIoDmbCvonpV6a{BVm#-m0S)_m0u#Z&~P?a~~{Z6CvSEAzisX73FV!fLEUH zNYk|xw?7@Ca3d`${OQ2_P6%P(%65|7BaY{PB!GI)ek_cAN0~l%Df3tYEHIk@SLL44 zgV^`%X~lL#UYMF6$I|;&SL7oZnLZFeZjKza6qGUHwI;Oi;eIwIvk#m_t5ETfBj`6K zvVBVmDSb&7=4KU7KeXs0rIqHf5$6<0)Y6K=#17%@v4g~WDpTk9N=SRcQL1zfb1JyS zR6Q259cjj-Y;A;z1)Jz)_<0un_6*BjvH;^>im@}NG96XX;v4TTXN?m@=;Dkj z&SQKG94}rLG}QR=k2jWpv*2kdkxj@hIl#BBFy#YZO@L`f;^@)bRJy!zGR@d$!&coFLe-0!G_SLqji1*5 z&dXKt_>&E26m}Er+Ctfkf!#15;Y#^^^I3F=A!W6kBL}l+R#hbm4vPxlaiS6@`Dl2S z%(~0cXD*`gkM~3D1S3k55kuWXPmFK+$ye3Qh4WuSiL+{^wMWI#qiZhdR>V@gP7x?~ zm_fTz0lPS(hEA=~q9Cl@>6{pCbT z2;M{w*Nai4jTqK>rNGcWU1}+*XA{5n!WP3UlKrenefRoVn7So4_KpIhrc_p&@RMz< zjG=&&0d!cghGzQfv8C5Ov9etg@Nrf!)k#Z{EdPex($Znt1(kI6@;tcT>col2U23YP z?;zA*i;w^6aHF1D!kqvosQI&&S_Z7x1o`(Y_tGmkc5foNq)a76l?s|V)XjSc#ga!r zCN(WPz?R!s!(Qi|BKI4p4v4GprGCJE-!8YZrr6h@2u*gA_ zo?ki1y}aRvKNK&s-^xa~J8wSzFj-0izBlQl?jl}B*AFWTchKLz3t7@j5met;hHHy% zkm90elnoM;{P`7`%)SiYyg#s+(ZYh7(043v-yi;pK|IEd*hj~ewqwGG?X+OG82*x| zrjKK{k>o`S7&ZMm$_@mg$J{~KcJLP+f4P_T=uV`^m$KQWmR?c{i)57vyI{hSWcroa zMs7xxuyu11JKnhjQ+LiJ=@ZEPa6ZMZZry~@yF$=>Ck&omnKU+f$F5u@Ns7WSdEsVn80NI-bWR>%4BfEJ|nc56~di9 zo=!uPcCjZO!t||iJWZUC4kHZSaE}gm!_6vF3LSE%tAhrlQmM*nmSk~)E!w0)Mp!AT zjAuN8F(AMn3$^u-VzZdr>QnH2ayAV7xytKZEwUxu6d1XF5-FuwVcNo{!1w-P^JG#e zq|E_LXE~9pT|H9{L!9fTM}M-@AhKo;U77KenT!qv2}3_B9Ndb!-v?pHQH#7Ec+e%C zDqf>j0TxYP&6Jzn;fk*{`<8ZOpvs5<~KJ zDYj7qawO5_h0h%i!VW(v3R2gn+i7bs=3yHf@lT0PjP$09AKx%_+DTIHH}QoY68KSk z4xUJlf%A{e$uVmyt$C$EAI^4z!`E)8o@|b@O=L*#{WLzt^bxOo!5KH$^uc1~0@`pw zi*@-C_@W_=MA}m6*$02Hd>=z|#A_IDcoh78b7XZw z62rvylT!D5#->!WezQ)lWk7-39x8q@G8VuGy)nfNzb zjCB^F{i@q(y5>f1z^{YtcG!V$1sf>!@hmn;bR@`kKj2+>ZG4$N6XFJ)aL2jNykxW_ zb~P`haP^0RHy;OC>F)scVVM`|JfDRJPo|QTj4!Q94Z&pgK)_}ycn5Z+^3E0MtGI#%0OqGs|&yWb2zv5?BV#397b!&Peg%^T`gMG5OUw zbl=W~%$mlK`vNhV(wGL1=c&=f*BO}or-y2B#_UqHtDt{_^F+2n2DNynTZ`>)XM-M#8+vD5$P2B@glvys=^(-D-8b*q?73cEt(J|M$aycqNaLnJg=R| z0@A#>n$y=|dqx90=?YMllEm~{b79;{IT#o2MYDy)z|M0Kn`}ITHnfD|sDB?}m&gS2 z6d6G+RXXI9Ih=vXEdgC|1+pzVOs}6>kjR)~`a^9HH@OMc9$yXD-adjItNduU+%vcn z(F;AlGT`!y9u_zEI`6rPL(BQ5c- z`wT@AjCGD1g+`l7$hT(+FI440b)VenPLKj!f7uAv;>$RVEr!hTEeDfA_R{k2BS<0c z2Y+Z)GsLBirqPY+)P{2uW1RMFDu4KIFZmjBBi^tO+8; z%el%2)ij&-k)`ZK7`0vxr{DvY(E0;9zU4qpxG8RpjiH>1_3UKxFP3y>GX7E>&s!=^ zXYuk?Ok_n2oyhm0*}i+|eXcb*6;38aldmjzI7^UjzY9i-{<6i(2jQ%PC$@Om(ZRUI zxFN=poT_cP1v8YX;HNBBi3gEv=x#Q?V-H)b>45f*hv50(G?LQqfbRBOepUZScKo+I zZ3*|rxJ60OFSC!$)J>($JDuoF-D_^gg-mFT+C*#X17Tk7Lv|o4j7F7gLHPTY-JTJ{ zAGwrE{53C%+mHzxro};PvkRTRzKj;sTf?o?EVe{*F})6U!XHT@q%;^|yH32Gl2gTK zYq>v3^~|N}Sx%^PY$nLh(j%1=d3e6nom4jHQRA(d*l(K0s-GnCR*Qp}!!v1`)Lmiw zK~x?O?GYu}3kS(OU>RLLH1eIj*PcEp+<}|KV9f6+wbQ{69T4^Zo4wQ z>5L_p{P{TbMFq2S5`m$sQ`j460jQ39#VVhLlLo5Lmp7+Kc~%I1cIX#WxTV6IYj*T5 zaW^!oTQS$LQDp9(3k`9zKt5;(EE7&+RdgE6jh6y^T+cZ@Eu`X6NnpSaL-o5SR`C6mUUOa?IpDSDwF#1DS=K1PCmY`#2^hV^R?!CSw_TxO#jj!2&dA4Sq&nsX_n zobLpo9Xnykj(uS9YY(h(66R1LoBO4{A0m7nak}x&@O1lWuIr2i$h}cu#-;5bH2gGb z)WGRUhrxPoF1#7s2NJ12xEmvy!L@h?_p)U_*Sg>&%sx`c$1g4d(Q~DoP;v{Ldmj$9 zBgSID!v%CmT8^zzyARcS#F?M6EW7BNg+~7=P~-s%DAZPDzg)`k#18@1HD$u1O*KtB z-aAsi^BO8&X^%Z}kZx*gJu)2XCV6-`sh*!wAUoO{@PR`Im}7f*H)%D}9*V@QH`Z*~WA* zVF-k->Z9M^+w4=uYP6qMg27I;EW$FHN;dURm+RjMn^!21>&2zGFXAWLYqFPiOzvQc z{%^Rr+Ed`t~fij1LU4oV|Absl_qSXrAH5=^oGgY zw{Lx*;aSMSoqg~@;y-qO<0tN1;s!L`FNt>Z(@0awoU+oE(#0iys5i$B3w>N5Yeh2~ z;O??lLp-M)s01r@>Okwj8O$@=#!iR_v7LTXxQVLK_{cFG?vGHXFzK0W##bLSGT#jE zZj0jyixHGFaw-hWeg^`reSB1f8LrLPPP#sptZacAU9CIMV9-?o*ZrfD4R{Z0z|kq}{MjPt3!6X#6YNQ|qX1Xt&8CZ#7U z`CFrJ)B+ci>o|QCJ#c$h3Coe11etS; zK`Ph*kF9;fCT{LVr}PEv;qgl>!b$;;r<~)LOkWDea#}%&tHRA6ok|b?;y$!2 zVKwpB;L(hA%=AkfPEB0O?x~5h>B7grttOe`z4LHV?-U%H&a+Rm`gkFU?YR11DT=Gt zVz8$q%e$vZO~N_&Yi~zL`?#0{Pl$Lt| z{Q{EVPR(8DdpLzU*Ban{&tPO$b+|uc3|1$GvEpHk@k^IS>w+#wxt)nAG4t8iM=y9y zq3`U@bq*cN3fPA0N?5#UDM_S!fvUg-?9R@u8BBP}a*hQ2Wy=(oib+|(C{M~coviKPhcij&3ehB+8DaS;sIy%rchTSBtyw3zBn zFPd`Nfs!jnlhXJklK)+Zxwr+rqBi2o??xDXWD0k3lMxQKUt#gUc-`o+Sh4R2SF%Hw zwImyoU27=5%g>;zNiw*rSrfJ{dB9B`3PYdSsoeUmskl((2e<2~0p5IXg5n}$*z3X^ zm|{1HvUhc3Z?zNKI(uz`(8jd4V@{VHyQsvPN2U(FQRuvHf`BnffhB(Q7~U%^I;Fct6v<(@5luGN^uIE zkLW%!jwFx9&~EJ{_WiyVeGM~5U8P#SY(@;oWw>Jr4S>z(eeh?ZA$5p4;ct=6Fkw>+ zof)_V!Tn-PqpBLeeO-oSC1T_=>mQqUC4yHkzsxXN7^OSC&~BX+onFvxE4d?<@|tZ> zNq0492p{3rx17VX`-kAVeKKC=4`XyeFTAkY!;ila%P(K7ibeH$WZ54;FN4xJt*;W$ z?eGp39-2hr8I{bYHxnoAet_N~WlYpS2RS8YGB%ruyJYLpd|nuFO^3NVt%kP88v}7h z(H(*94;g{Rm3~fi4zs0s*LKjlyM<)%V;s8It*0?!x8ZNJ6#Jeu4fj>Zv3-5M zG_S1-RXrb~l!pYZNV8_I4aT6bHkCnjIfm+m@X2Q*T>r3xC8}P783m$eY=61U+AmJC~!~IPw7OmLKD#Yt; z(7;FQGGZHW{rv&F;Jl#^N`TEb>7b@Kg7oZI}pt?J&aT&FW|u z@&F!=Ilv4z{^tBvY{OI9Gx6|?4{T+r6MZ{Z2In5A;p=QuHn}X4^lwk1OV$|}Z61Vg z+AX-a+Y4#QFk=zWu7Dncz)CCzVdR1YD)VcH@RDtqA87y&8`7Dqt1*srErVIV9nd5w z84k+r!(A2vsJQiur;qQs{}#}AqI63npSC$}LLF-2g9 zd%27FY(o}Rj_QI>!@lq?*Tif7QbTd;7tBDCK{c0bJ4J1X#Vz*ck4uX}^r%ACebgVC z?q^f;6m7h2pN$K%``~o)H9lf+3fAc5;k~sV*!&0csoC@;h<>zTEA2w)?o&~maN;&h z)L2Y9E21GwH=S+ahj+^WPg-w&83Hzjz>2A}SfQK;WtLl@;V6Pag#gB{3d2^vI*1T@ z1qve`GVMAiI`6p<>!Xk2ju}qaDI|?ImR*A27HJkF+XP8o7Lb>B58iyNLGQOFbbRv! ziCHtkOYrTh#S_}9ZEi%Z~==@{;`g9f#4^yWnO8!*+l zR4hLbgcTuNmcG3=Lb%-C55N!0b2NJngAagwy;NH-_dBoG&+22E1f*k1P0}ou%&W5 z1)sgirtV$>TJk+8b5H@Y)@QL!;eHG#ETCDPjm%2A7;bCa0PeZ~bv$GE3-?=Kx~MQM z*l9}ABcd_2ZzkIRcbYyE*$VHSL&~G8P=DqUZ2Dsj@l1t$TbubSC#K=C7-_0cK8TWq zzp?-HX8L_JgSS&%h6fVNFwJBWO7UOW=+6;g@H7*P)c=D~pNnvj*HnDhGr-q(yWf-}i&Fvlwn-t_*4<}hb- z3{D@b5}0^bvO;-vy8Yz{KXhjgrcP?WD&<>zwyqwJx8E?)+tK(yF^JzeFoVRpEMRYk zGFqA4g1ea_7$me4L+lDbZMg$l46_+}@p@R_ej8@6TJFNMcu3Ov0e_;Fa#xF|k}=hx znPd)Z_KU#=*Dq|vb9embw0C$%Ewg=_=*Pvb)@6y)X3;)yq!x{Oe%s!EpkKWh$JV8w zaCZT6XJnC!Sx0vtEW*AR=%21%V2{*4;hBIu1(Zu~9n0tO= zua-9p3_DFo-R2A5vhEnlY_~*3r9xK2yK~`XrucP^F$Ry^jz2~0XlblBm@jLC);x85 z9i@YtM1O$cJ1_R&=MScI{s)`o)&y@$Dp=lKd0b_0&HA3N!GB$5RI8%Gte1X(O}+yV z%#Fvf_w|_dVPO_EqY3mLs1PhY$@(Xiz|TYFyhM2byJ8x`SGg!+%4!)Dh(zOh{R#Nx z*9DMOT?5M7oo!3L?S{9L$D;IiMLhaO5hV)y;N8|jX246}t-EFL?C@21bHx!wXZxV_ z*hrEuI|@GL>P!oZvFUyt$Of+AQyNp)ChJUa))6Ha1zQSmUW~#^g3$4t2G-AY#oKGl z(O?WQ)}~GU$5*4AiWICaPse`aK)89=4-Kx>lGC*cGT)zr4SmgmD>4@3XSa>b@w>>3 zb~V72UIWzaUxnLUQus|RC)pvHSM20hFECm<8?U*D;F-1vsyb@RrYEVQ{1Xn^Bo^b$ zncDQYxwZM1ZXavSKh1p>SfiQqR8sTN!zMex%%Rn^^}}M4x&KF?C$$Ck-@V3GEjuev zWXW`Eo*C#k9-@K;G1#T#gk71zaI!hsHczbwl(SEg=)qQ2!VEZ#d3#xB$P50|qfWNx z<^g)vdyJt+1*^OvLXBNKB{XP|Up1R@B!BhArp$r`VX_J=JLyVkP1$##9#J>UNG;NPJ{Ngr~ zlc^iMEVsm;^6&WNs!wcnro^JU{2AtRVjM=td7y!$7A=`_4ip~r+m2P%#Uqm~$szXv z>nJ~sSL$EGtNS6iDsU^6PV>VWT{)Z_J_V+h6tU#WW@aY5d)U7W^FMq5{c6A9Cf9M%&Svwi6WPkH&?N{)_#UwhV8;4O!D=1?ifV zdX|z$8q#azt)J*J9pn-~K*uU)7s;4Q>lJOBZF59|Muh(lgl;9jc=kX=dDbX1iMy8wBwi1?zUvoESiAdg&#t;dlCOmeLqfp z^cp6bxZ~?vMbHzSg>sG~n0++JXk|MR*4|95+O~zX4w^T?p|b^ytYJG2FCx zB)Sw$rGlr)?Dx$IH0e-cj@(LepK%fo4LyMLVqw2UfMXSkI8&40p7VPt57pn=fos;cVDE6By6iz*?i?PU&xI3Nexd$8zk(!adwuhju95 zJQ`PPsiJm9Js3-uLSLaBUTD6EgTw4*;iAA{eNcj%8G2N9(wKUl9Ds|(i|F>Q+kz|Y zd(m9D78gWaW`E{N+q^k;9rD{FP;XHU)`?WuI(km#4$K#0!mkJh-PhqrPL%aO+71dr zN>n(`8_paSwY&v9;F6TnDQg)$bHqrR8`HumMQ zl2}{v#|Uh{Il#mY{NP`IsD%0pru26CLzwm0g4VPYGg0qSC_BCdUp-~~1F~IwX+?7abW}QG`8pJ!mBl_GkC2B7$BOUiStV?XBK zVh^g2+&W|_B@i%n*%Vl15rUPCJl7cEf){+xSvS1cz*1((p!Q`Mu)Es|Vz4{_o0N|8PtCFY2v(Ta17Pa&F7*fcU z;cOv7iq*QE1ul3mvuX>)$zNyV<+foa!9@p09hrzHNt{`HbmLr=g6Y4q&1}B6I)1k+ z0-s^7YISfA+q)rwPF`+jw%=uG-_mW6zQ7Arr7Q69E+e*aS2-l~M383ecy@k#B)e*U zl!e|{N-`BXSUfq4_TA5ddo|-Q^Xw7s_t$dt4w^u*-!8%0=3po_{K8VjUV**XSxmYg zOJ9Fgk=cREcr)K0N2w&S+U|bdnm-QbL{{V2yI*1Sq%>U8!oXBym{VOCh_gjRsK=}p zjV795L*OTV=<*ge_joJA1UGzM^O;>N@D->p8NRWZXoTZOC7|itA{_B@EIEvEMjhdE z%yQpeFtOi>|HU_8?(A92d-@n!wq-Ql&zQ(dj7mZ>-@?}3(jjNHXe>OV!BSM-L$yZ? zcFY>bm8Z`Dsb9<3(x<^J#>pBNQZe504QC;zr;>%>5MC=&rgIj1uzqI^`|ROCa}rHh z=cp8zvRZ+@ymsZ!JQV>{u|comg^;iFPtfy_*{0h|If%ue0evX?nqt6}iDX1JGM(dEaT0pa-%@=UC z?PRgBskl2z2TsQ2lg?%Zy6Nde*#}JNpUQON^o_VRmqZ~@)tuc@GC&yU|#$Qerx}C23snreCrC> zCYUy?Ym%Vsg7=>ahBrWe(;27pv7!_y-Vat(|3@HafRv54z>^xv&?dY-UP@c2YA zmadZ_KBI;*Gqh1wDj6P~{sE@*cafRSa=L7m0@h{P%!1#+mgywltwu=B``>Y^0U<_`Feat53ti{Ud zJ^bSI`P3t(fxYLBHjkS=8b=&G4#uAlcLujJ7vCGKB-$HypBVv$ZKrVIuzoZc97YwN zA(lEMK^Np@G557Hg%5WFVOCM(I)5hiZY{)9U6a{dAss&B#A&dN@PNz_PS`6J3T`ha z;hGy~*@sXQ;Qy-NQtb_Rah=fbCd$pQ{|}rDH8Rr&Z0$CN7X`hdxflKk~zY(aZsYgSrM`HP-MWj$Slk0EqWtsV-DMCYoLe(yU z-unTF{8Gx#h;3x`d9fH|zm-*t`oIqL`Eo@C5|m%M9`zQ?p!_Hy%)7S(*CroFx(=is z8o|;rm;_JDaHNO??5qxlrjWXV`I|NwC&o zBg?rGiHRDa5PeS)XAR#u9E&zX-^`bo;FyUaYuxd3P!{^UQNo|6$KlLL!+L%52uymD zM)Sr_WluYwg2XuqEUZ(;m#RJ7j18i&a}u#njz{3$>TEna+&g@3{0Cz_<(ccdJ>a|d z4{J0@qXV9XOx41HHphgqBu`VWX-h6#>@DTdw1rJ+u_S?80Xm*N%0`V`PG=US3&d*7 zP&vh%k{!2{oyiMD*o~eKOH|i3}`A)N1+*T zkzUlUE`e9|k|?BU7j6|SL_VDLk4_7z9+iA zoQ{L1VzKh*Pgba$KpKIknWjr3{pmgd|Jq%c=KQhPDt7?xncCxwl)L zmx<%_snh6lT@qv+c?&`_@3HxZreWO=TY9_79*aAUVos?kc4!`AzrqglZ(~N;x{lDr zA^+!)qGJF9RfnJ?#~c2tjU@L6XKlavO~K~xsuUM_ibaNcU~%UltnEm_w>k^?q?J`5 zWIGNNhAKG8_Cmnm=eF&Z7x>exh1|+3E?yfDgx|Tv=dG7QZrmc0Ntxzt^( zCiFho=yr2T$u`V6NrTJH>SP-q&*WxW>Ef4L`P{2CJv58U;=)dxXLE8xc=YIGcb6RG zrUsVsZrvX_eSbN;cv2fQC8eNDv7MV}y^ZajXd$Q$9*MV)Zs$zQw=x+)4fo9;kNpo7 Cv?6Q( literal 0 HcmV?d00001 diff --git a/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json b/demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.json similarity index 80% rename from demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json rename to demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.json index 0f8345c..55cff60 100644 --- a/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json +++ b/demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.json @@ -1 +1 @@ -{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,17],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-agent.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[17,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]} \ No newline at end of file +{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,26],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-road.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[26,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.weights.bin b/demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.weights.bin new file mode 100644 index 0000000000000000000000000000000000000000..b0d7df64578a25741120aa2c5578d9c916ed7cc6 GIT binary patch literal 16132 zcmd^G_aj&T7nWVJ$xKGHR7SGiuX8Jrgd|e=R9Z-BQA#O$?^(7GAwm@H>%2*&G$_$d z5ou5<8b)9L$M@&^%e}9AUgvqv^E^i|q>}U0_$Ikyb`f}Tj}oo-CY-$$mL%T86RJ+% zB34y;;6JbmPipbPWUw?OCnvL`hS$i6JtE+9+8>@2_K<<-$E=5gBG}D~f)I!MBz>JM ziK@H_gW>kTmK)>a&MP#)T@&v86#$+nS+e+Z1{S6(qP>bJEPpBp`HL*6_d;{2I3X_AanN&tqhv+X{k8i%a((|vv2-UX()pb(fQI(AA(_NsGrm+Xs2oTp* zp7hwxMObPi265qsvEJz#*)mxV`;&_}+2>PW!O|ou>bQ_KP*;VX)BLm`{2W%Tdr2*< z3!wV553#y@o*emZgJ-teQ}x5g(R^k(?kn6f%7^5IDZIr?WCl-jC5TVX92xfar>zYLQdx(PzoMu2h7 z16X+Z4;@|{0iMe)K-Jr3Ry)H0r)iaPp8i~iowb|T@I`NF$H;8BZJtQ)3Vb#(FRUS- z6=p-p-gR^}x0zOaxl6Y7J|$PZdg!vspR~3pir5I;BKecBPA7Nx{zj2g!x_b+q^21x|uQ3GoXRgKDQpYLJu4cJ28b^*s{@^Tt&np+kd(IwVunT{EP-0nUb$z~wkCtqBBn!{2Ys=yI}&&N za)$TqH$XgXBh-1zGY=e^=!swF!MgYm7=0}v!;z~nHGLdj__X7a15dFj!i;-HJPtAq z%c<$MZ5U|Bfk2KZ*ZFq@F?taVV?>V$@`)j-A@-1-VgOC~AMjb?YeIgvLsfAoN-sG_ z(w|1qzh_@^tcFd%&iM^pDwBaX#P8vPS4PbE@oC)i7DXtyXDijYoCe>`jA?L=EO;Ey z<_4WT0+u)B*f2gj_`_cehq8~ttf}We(^Tlv-zV^lUlyG4`^et>8Vfs?RFJ3LX@C<$ zxb$o$^nDAVFAWJ1sk%>_LN?G5gH%-T*TRJ6xhOxsik!Ij1~Yz1k$_im$Q$vJG+OS0 z9~xqo<$#gscedoR8(zLIYz!Naptbr28nlQt8HaF=<6lj;fq@6Dk z$o{35z_w3=#59bM8J(@H+Rs}2>Ds~0&HX|?56a*ji!G?xPzT2aRWZ}c5qCz6pxN^> z>iQ@ZCjx6(PvKT@=@Z5eyDnh#22;-A+2h2wArNnM+#vP|$HB#mhiNQVVHU_R0)F&q^Nd%UmBi5iQHz|1lL$ zEjMMnk1s%x2Q{eEdKs?GU&cM}v6XAG>^g4W+)rcNBJqrBA}F4ffRxAg$mFB-Kn7|6 z=kP%0a3ExUJrAkjCfp%yb-3$rlo1uMgrvqsYVMnW4=;ZpiYqf9ZzLSvDPO=|MHMu^ z&w~O{l8mhFd?uvcpXvwdz;sRvxuNHQ?Gd(&(0w`X-2WcaU$a`^Qk^*CRd*I`7wF-g z8=E1#e-%s8lwy|xUj_eW?ube$Zt%Yy>Xn;@3w#GbcS=H$z_ z)O}nMoqwj03rB?M0UH^ZvCav^q`HWq(OS^H=SEzc-V^Viw#?SseC)g1M)>l{3~+hg zNq77aVJwq&qnT4IjO!S|JTZP|34CC!A_nP^8FxVV7lm!c3UJGH0dn<(xe4<<;qOTs z{A+X<_6}!4LHa#}q)VI~7fMLp2W_HIxCR^Z0-)aFJY4^jMs)=h$dO6(OMV8jW+nb@n3i_fURWUDI`t!}0> z>TT%{y{GiY{&Y~SF2R=$0WjPA0?gO7V7c`#>7(!8V9&;N5PZi81UwXI(d0C;5+5t*Vgn;+EVCn-5;I6lPd~1x)rM!z zAE~w3J`7nFO;@NjK;4B(_WE)Sbo1GOeMis3$L8x0l*Nw=Iwi>K5I^YOl8s(V0WLcX z)a-k-tmcWV|6$|-Vf})*O%e0whU-BAi2}I32R^Ipv%P?kaG>D$Ar)0 zIln}>c%gzCtnpyHbot1$&GSHGuRZda3vx$lwQ%%?)yEMm2J_np9 zV`1X`L%6c8lK2X?;Xn0_ut(a2oSO3je%cj5zrPMF*_|-e4_3gEwPpAu=qpX^-UPj| z2eE&9DP6t01)r;AP?<(g+Rk>7+=5_Yd|8OOH2pu2^Zg9XQN z1+d679N3`%vgpHea$P10KUQWoEK1CRIe*@94*a@Lmb>eanv3NqlG{clJ8dWvzKS{| zIzvp%d+53pM#NThk}r!M5UHFg&cERj4OD%^SK+J$Tb64gzZaD4;r&q3_yHvKLyka1{d2N74^f!{b_q0K$ zz8eOWIB3O2auUR%Xvl~p7_Re0liWvCV|yG<&+>r8R%;x3=S4#-(`iAeH!2yF<6e>r zdawzF*GiISZuuA%76Fxp$;i{Df?gSWNrQDVMqd}fp`#(}^VT@rp1Ymk(Y5$MSRb}1 zHqrHCK_Is{6f3u@==1ZdB@OiLFGY3uWPk3VGOvYeQ zBYw=VBcXq#KyyVHyYqkv2|pAKZ+1wcS4@kE`l;LWpS}a^_f>)2O4XQ_U}1Afi(g5W$QT>nN3RTh{* zjouwo)1nV_7d>b4wbK&T4QRt|O=E0q3ZZMLH9B$6(g?HN==p3TRx1n>{q}jdBrBTy zK9r7qzTc@3qXb_KX4C6U=A`WS9z1@ehw#%Cu&V7ddA?o^KUv7bsek9G)F%!)=@`J!tFNM*kfh@rN?XN zpOXrB_4{d@ZJ5WZos#7^9l1=)jw{mvk5@!n_d4zj8mF^@-jKid2z~G1NW;%=gsZZj z$@@lo5Li6HE`4VP}!mt(d$*&7@6j@a8jV9i-1U&}Yf$vfUG%ij-zsI9=dS*RjNNF>2$~^d`qo2)g zE(8^J4&k+!!Sp;%ppSNb#jY)Cdofd)79rtM0qz7!z(pQ+K?gs>{^XNQlV7)y&A{RU5t40y2Jj=lMw&N2Xb2Zx!;b=g2i$? zOz^@xw9Rrp7Og1d#2?;^KMuBVG!7V}*`-z>AC{BCD<@(4a4=d_$#bs%-iJHW!{N}6 zQd(s>6C=kJx%|Fjm{=Zz0i~NjEFc(jMWWDgb2N7I=R&ZiB*uGdlFmcH^yv6X+A8M( zi}&l{>z|n@kY&QX6DkNBxK}``ZxT(dotC!sIF5?e5CKmxGZ%j^cX%TL+Xg|5rn+h@upU^=yJ2*NPf(pC%xMwzY z5I@P6ptQc2)gL(wzEz6o?JU6ExAhHY-5*|hT*(Vwjq}lAB}2UbDHt@HV?ZS`7_Ol& zYHd9RpS|_5dZ!rvbd$mbu9xWB4HFy-&sn%5h#$>fRdWop`^eVngoZ0XL9g0nf_UqvDu_9O*dWnj`YcN z`qU~0z2GV9KX)~buiZ@J+9+Gnv;gOhYqBHClCZ^rA4L3)z%+x4WZf)f?)s~W_+z4p z4tr}d;;JiQV!J;StlUf-=jY)b`6&*)vzk7%*T!ef1d|^2A9`08PijaI!BMGY6%ZmMuJ}9_ddK?nmIfD>m@ti3$DuV=o*eLg4Z# zh@DK|h&eW=h)en#D&_s2xanx);HGS_3yLOeZC};EatczmreJukh?BL8m{ZSfCIC1N|3?%bGAE)31#U9;L8EY6t#)x&bAm zEn$I82#$UaXs8v=p!Rd4>K{DLp5moHNSQR6&YnJ#_2X+H3#-n-5^XVDDfNu=qr#a! z-k}E~^3AksxeB#;qX?4w6i{C@iuUTfW_uUyMtEGrM(P}4bE~(|voYy(vTFgJZdrlW z-C?-Etb{r#M8OGQsoEZQGH~(=75){9u2Z_wU`-+2tUASO$CT)KIcezldV^NZEu+`Y zn&Wl@T_&Z?1BS~iaG4B4n`JB69g>b5)mSGEt&InBy_N9MWCj{unI!xVUC8pEk#t7h zZ%Wm!Q=U0j!0OI62s=ShGvqB{y8R(_?LX3a0tpkDPHPQcaz?}zh^l%7XP(e=y5Pxl zuv>l}=DO(Pw)4_>MdUEP>DLD5f4bbd*Y@x*F_80C(+{4<+j3&tXTX<-FUaZm=|s%K z3X{I)gP?jSTq+NOlGYITX30m2O%hn!3sq!+a3q{#m!MYG4a(`54pQPyAZL)xE-&=} z``;{f8mNQo??EcAT>~9iURY(j9&eXirG@-j*cFfjba>N0wL?L`wD?)upfRl5TO z43uCOY=Q$#&hRL@k;Vuq;D3JZcw(Lr#{IIytha(7TpmoKKjqNzJq%jEjsTa0nJ{TfS*!h#Q%l>Sa&Xm{nMvBfJjYt?aM`|ArXudl`G(CoID1tP{b`-0kF*@ojsgJ z>$;5ENWo2(U31WuRotV4!s@Z6HnFa7CwU2`t+pf0v!qB|*civ_iY2YPdY>cnq@G5W z+QAEnK)mUw0Ke?#z~X~b->#cYA2D279ef6wwuWIE6G8aK0=;dXBv4>4tNz4; zv@FFZZ-Gxz4DHa}1U&a$^ez4# zVH16x&@Y%tUpzQbr!4b>JQ51xtXvRF&)ILndZwO!b`XO{3+Ez>TAbRBZaguY;HA|@ z80#&~%1CV^Lr(*sS!y*7Er^AI3R%wgu1s9_SOz8dV{xZ<4O=5-K=tHGp={0;hV>9}{CAV$aCS!rXQrkr`8RDp*oYcqXz+0y`*jlE+oY5FFK*z)$ip$aE717$EH2%B3P+uX ziR>sJ*znv(-F>lCc|ZaJuU}-lq%Yy~cx@DI5Fo4KZLs#*7n1cml-%6FvhNQbB}bpW zGBvLWW{Xezn=U-HzCrFo2wkz|8syP2&PmDnWQm^+`I(bS-rY3Ai{ZB#V6hCXl53;m z5t430x`t=4o5B{s zU7e$#GhPj^_PxV(Aq#29xHq0Svko6M%)-;-qXLvqFOVK`Pud({TjW_AcalrvF1DWi3YZ)<)*Ff&f?QtTuP?*3-D~;Au!mkYlPW zhJY_s0=HeC!9>{da{HQAp|$rW_K$fY&P@{J8p$a!`=>BXYU^z74h2!Bvw9Czeksd* z;4xw3kIFH`MvFOM(g5enqwz#o0dUUAFeL3W{JNvVL<^cQs#X)Y`PDe)HVlE(f3ui% zx~g2}*lhT0)kem5h;jv^Uf^)gPgqb>jz?_@(dl0!j=I#Mc76v|L@RNle~2&^7nd=n z33g2BJV~a-^glQ`qRsrj`(LHbZrHSU5gkn4$z7sw-ZcJP3ft)Z80tD%G(No;_>TXj zm52A?$`?23U_uJ4E?UoxU$qIcOv9;z^nM~OBE)9TK98v0!Rf8|K_ZXFaqmS9(VaPs z@b&LU)}VHP9{H_J=JIBehf7S!G#M>;zFq_e+OlBurW$mcnM~c|3DP}UfCf9Y!8bi0 zKfVp3;T6m9{DG~&6*ZokTRY&hFOaX#a^X=#JcvHN1x1%d8L<;fVJ7b{*yy_qX-_dJ zp&LMZ=_Nei*M>*A$MEp04X|v%FeySWqV!JyW?pWg>xz76PS$zy>$@YG-7$a}i>k>y z+W-tcW5~_=WyFymyGuT5nkTZ`?;QKqTITq8}>G{Mg1e4kL?9eLsZ=(PwTkHj_ zsMrO3FJ!SWo|iTk1dt&wO**+alw%Pk49$<}6sb02%*8=*U@MG%iHW)hpoVvchOcX7bp_f_|==lnQ#tp)m9Lt1go%vp>nEz-f{P1$6!KMUk6ropG1 z4>^%~>2UCGB{|qOi!)>40ZwB;E@$Q4<*<5IJkp(m4RLw=oQ#+%s@%GZ8JZsj8z&CH zG2Ln)g))&V^o>D`k~05%th1YC=9oo z1{U*Ql33;M@Z*#`8b0G-NnI(9dl$pZf4O*hbvq8%hA?%!3+p%?p6F~|&L%(0MUf*V zaP#hQIF|eg%Wg?CyW3JQ+MtDGoE62l9}VfxhD9XN>>T;;4@(Mc+`z`3(z|OMK%ajL z9$_>vJp2u*fDv-{ZX`Z5H(474>(doYCLYz0RX!#7_J|S&Nih&1ag1o~GRKC|1XRDkz+12H ztX`x(C=Y!z#;@lvY;GRhXNBQgSO=DU_P~Yv6~Rs07T%mW%jIIukj7udGZmm#{itm-Nem)t}) z>m?n>dm)hZf~4&Wp-)U0FFm`CXB=)re1S3dgxd!a&wrdv_bi4DK^yUcSqo@R`v(p; zgqY_jfbqVuc<~j0xYrPwYr@N@lqi5($}iAfB*?Ts5ofNIjFG>U{cQb}?R2Mp0Q_-O zgcCmw&~5+Sfn8gp!LMjPe8^t}huhX*ujD7vg4^-aiz2i*upCaGOhW^=#Y|(+RlK1U zgPO zIY!Gy$(HNHROe|lzU94$TZjBf&s2Zf=>Hc&?n!YMGz)>3nm?y5g$MSm;4xkL<_*xl zWtczBg;_Qu6i@98MT2-}(z?YO<#_Mo=FaJi?}A9qhv``y`;HXYc`XnZO`ikPpD*Bg zc$m}Z4b5oqp@6m-Q@oxv3*;)N6Z^D!^bUIkCi2(O*!voJw$2yMzevQ8OG?~-rcEU2 zO(x{C9;jjE!mVoF3peK6rV06w#i*RD{%sNKX#c>#KI0%1mg2&T3*Q=UB%_@d%0x|jG6 zblpcD?C8U1K}AF}FbXcM7sng1N!${nR}f^k8vJ&C1do_5@SU@hXf$7hRp&mD9+|~l z^(!?%UgrSsfg1W-A&Sa&w^MyX7Itrc$cZ2;A;p20d3sikcFs4(y?PUbDI##Zyb8ZX z*5gcwhreVIwx=9-`tDHQ;#urvCBPQQgB|CX-|>3Tq=H$zM?sZi%xF(obI?p*sbudB+I{Me3=mNK?*qDN+$cWVqwWq0p@^mA-HV0g6DQv;odKQu>GAH6C3#uGOxZR zM#aZ5ASwfG{VU;C)gUX8FJ~GiGLw;S;$z%)r)Of(^-&8E`0S6T-aezsf5&Lz$XrGvE)Hf)S~AzC`cYn6H_gEoIQ>b7 zMscEXrO#`SP^*SbrH1tT_k;B3qJ6M^=@Tq55{IKDmq~gF#pf>)sm6c~21aI3zq{!m zu2h4+ZV*;(TP*ZDCvv4W#Njj5S(qJn2%=x#rZqRK>58Oz)Z01~6Mp%?>)map5%(ME zLj5#)?T0kJ*o0I)DFYp>W-(!9(p*dFKG+#G9acCL(i3qf(CUE}_dV|dI8(J6Cv4=o z5B=m&=2$o}FN@{Ol@Nj??Ge+RZp}D%jt{<&&16h;Wydj83!K3bnZKT@ZCV0bd@p0kk7Bf5w4cf}7lA0hJ~3-(G!1rfd%(Mek;)PXX=J?5bC5oRkmLT|=x|^o&Xz%wVg&EN6n$ z$LI^)TD(3c%vF#7jip7m*bhdUD65c#x^p~G;FpZ4+#4b0{r$)(Pb7}Aw_GrLZ~&4` zUeV32b?lzKVvrd>4HCDRqP@_ZhOO40NR4t9r~bPzGj9zaxBX!PS>eA88arz6QhX}L zz6+n48MEk#?^0lr=>&xjW;5Mi=djxT!Q|wZ|4{Uy@{~VS3k&MyaaHj$vTV0F&i|u> z0xwj7J}8BVYavj^>4AlX@sQ5L$M{VbL?h-YeBn37j8<)EOt}JQBr9p;7hRnD&EsDRd$#h7J(vEiDxGi;dhN={}ZK;OZ9 z7;~rCvVRYFA1R~UYhlQDe-H6A3_y3rk2;4ZQIn#zc>7ByDytO3_WG0DXM7dwj>47?3Ot|Nar!~AmAx4)>MEK*+ z)vxK_OcSol%W^nBnh7UT254&3EZD!fjlCpy6z~6fP5+z9{)XI=#N93{7zfETP}dN| z1G29gb|;i_W&$Hv>DBxoBLY#rOtK0^j5P5H}nT6B}&MQf)eEOpBr) zw@;Ezc`sUF{S+$V0%>A}I;vMZ#GY;!X7X|e+HOrEse5&@U|VCYG4>6m2zCRdj_rCAOMRSJ;|*I zb-X;-z*)@~0&>6HQCLWrNwSjxv-NsR<8aM9(h*-Mm@gRwDx?J(oz;Hy13+ETDc8Vf6dV^YG@_G)y`ZM=nf5H?=E=! zwHscqk0v2!(CSS1GL z7PX|rAQtkzpM}Vij`WLHFzP?&(2$B!a)G-C`<2zXkG05Sii5FMyT3idjJOx=6jTIYbG zC;O0(dmQ+Gdtl?D5v&MU0N0cMkVSs6^f7Z7u1 z+MxJUPFCM^4*WbO2kr?~)N^A4T3;E41XUsK^m&^YZhI|8uCK?_6N~7J%iVb4jx}cO zm7uTqzG6v6A#<=Ji`>8e9Bw;>vJE*IbXjuwt>98>~ni+r!u^XXnvf{E)PkiF|Z zQMe$2an%v<#xVnY#4o`NnI%AOX~X<)(Xf8ACbVt&z+ukvLYU`1us`&I#y#0j?qv4T zjX{Z!^YJ~YS8pc*tIKKZt9R_biN&C$RZngyJ*Iwbk|1Ff1KQ(=(eRC)0S;tal-{gC+vDyUlEF~Gm#j1>YTsnE}^#no}DPaG4 zZ=9H^$+)zKW3fXFnpW^Luj-7MTwVpn*X{%)7LCz23ewCIl?;?q9s+Y?0dAwgAIOD^ zn0DV6^#b#-#%dXC*m#eoTu_G5J_R&7la38Wwe;}tTUfNJiXIMCKsb34cJEz+VfC)? zRrqQBYWYN3vT_j0E?z^2-V`)=@|Qdftc3T=vvI9cB05w|?bL`V0dI&84wDtQ%TbN_ zIB*neaz|m)^+@#iz|YlJ%%(K^7CbP#3|igNjGSsJl;#IhS3z?+P?Cy6$96zl&rF=h z$Dn(275Uzkf|XOg!m#`yH2*7s&%R!z+?Fbo+Or+`BSYCi-EvqwLj&BuCE&kxhoJxX zC%nxD!uK{la_55}GiRs_*SzCl?!E7)w>D3(zB}_FOYHzeAxd&s6q2o+07{?)gndH)2@-}!+K!nsknX2sz1L$QH3jyVifxo*7zwX(D zA;-^9FQ>IIc5^TAz1#!qEdw#F*#mEVRm78e?y#3T-T$IA@Y0T9n~wl{zB%{Uv<;y`S^$ z4<87p6k(3l)EVDV0A|-l=A5cLt|+%+zNlM4!);l5r^kk7zceAO2WBFp^oc$&yib-q z<-+5;wzQxsoSv8QB7XP#$h5R8nD1MNuCtTTi)WH8jj2UW=|fyNo5D(uW*oj7GnHkE z#~AlQvRHB(^VDbuESlxSy!@<UlPX`@0 z?gq_W)to$=Sk&141IG(aLtK|IwY&QP*e6TyxNH@bR6T-$sjakZ&=KdnTZ+onPNd-4 z6C$|YBY%KA)1jL_;KGbJ^fseS^n3OasBNLJ}1Oc?z}kpwZ;;o0&MZFOduV& z#58akpVOl8A>>>Zz+dLoxI94+XDB66gG>6z&zVP5)_B8~J#ElYo{W6X^~gQyNNdvV zAu*y6c6q45Tk%vh`GSzK+X+wQRikLy4RR<|iX-iN2r{okPT@!=2=&c{f@wnF|Hz%K z)V_;Xq%vWyy(F>ueuVQ~zzIwb9ALs1=)&e7m#DFj9z0Mj1bNdv@bh5;)bohJ(<{A@ z@N*7m@WtSR!}~z@@x% zIr`#5`kbg#M#0N7ku>xE)NV`oGH%!2s|b7RL98(t<=#zY=;g~HHOrC|u6a&eXP*L# zlG~^sZv)3no#0+(5Bc=V8n*VO!n&53Y{1PRN=8GdikU3VSse-ToCFX|dCUpyjz;~T zIiNFn2H%F*lPg#4xEq)lc(q{(gmi7il^+vu{FWuuJiAU)&(0&Zf7cK^;A8EkO2*f*G(YP~3f;tTUqT#^= zM!j=jrO*U*i;IARUlTCny*1(R$kXbGb2Ru+BA&nakCnKVhQ=Zxuw3jgeo)KDMY2~x zdzlYbKjMLDBMX^Q%g=CT!U?|pSO{x=%Hip^O=wrT4qow&!_se#uuH826WC|OJ7ybZ zdNkoVgL)J{Gn0GL$s0qguAryD6OQ`!X^DxLlt=pQ%oy~?B58V{1j1l zIEl>jAH?8h4W`dUkb6+>7=2>xLl&;yF4AG8u81d(aoZc;TMY#(ITRo)R(Mw^Q zh9~JiZ3Mg>Rp3Ui0IgME>aAqpnr9xl?rV*n4u7Hajx=sUFS>2L7CaPGK}}ml&V?== zT$0ehaT)ePyWu#J*^*3*EJHX8q_5EZ25HcA=sKJe*QWk^USr1cF$j|%gn(I!jJ$%V z$@A3)T_OF~F2y`F$m2LRY@71(1*lk(IipfFMvAJn zapl7}476xP`GqSmrR*HLCN=;=hX<+ZKr8qqnnUF75~!SKN*#_&!n(b#z$16BepcpL z;^+Mp@>_B-;Cc}dmV>8{SmN-6B;`$50AqHSfPPFM{8xdA9k2kYdNn4wqX4T`y(P=8 zS%81dPh#)bKzF;T(e*jE@RF`F(=arX89AAP#s(|kj|(*!p1S)kxwh=jNj+E?@(0(b z&SIiH1hA``u$^lX@vfjOqpK^-jp{B1bt9zDJIrwYz-z+&wT!DWSPtG+^57pPjW%fk zu;E}P)RsD;YE3GudHM?7_Mr<`JUIs^W~p<(iHUOK$vUQau#H6ICc|J`2kor4;Ydk5 zfZXe+=)Uz@Bx_qFK948_=~?F?+_xFNnV2&}7Sp&V`D5^?fjxKf$Xl|&fWW=tx`uVI ziPNZ@MkWh{K=0ZO`rx)6oEi9udGA*N+@JD`yt9c{`$G6vK9B5kS3$kj>EujO0{EYv zfEPC^AltEy@Ef?}tfN+>v{a94S~`<`W~R!WArVDX9{bR`Z(;DnbvX=^A79vS9ge3u48o%BW!7Ptes=cJikGO})Y;m+5gVrx|fi!7rkfDi?`R zVd=v(USog`WypbT!5lEO-++N()$G3?NPlv-k~6D5(aLGEBxuD}Y~R_>{tF5u?YBHh zn{^x+l2%5o-9_|QwF2EW{~(G7e_`ivoLHTEGq7^MC|Vu5#W}TbBOP6&hVFc8;cv!q zc2sI5ofK)H4!jN!RbvTR+diV8?@Vqc?=LJcyNP_GZk#wH8Iby83}+Ah!c7JKz(y~J zRa#+OW!rpU-@0-gZpuS5XhN?SgSs!K=91mkkSOiMh^>r;G-#CDJb6N<)e4Swd0BNHjDwRortlwD%?@ zEh%XaO8NQydG880&nSQ%->s~oKG>SHgk z`00-aHKVxUnjMB*eZVgZI_Yk(y{OJ94yO!_QTx+wNFLu8E&sG~%;yE%cE|@bKc(^} z#SQp#eIzZFTFF0p1CL*6&8G8xdA|P$e0K7fgK~v62ilB+)!r^p*tvxabJY3TayK5d zPZ~xg6u}7h@8s^33?Js?!iU%%Z09Ipn@Mi`d+r}veb*AVyU*f{{XUYnhcsHcB#Y}- z{S$I^uG7G*xavo7SLk1B8LSA;<^t^+Qa+monkVgGYqJyWTs)7)Uk{?f{SM^ppG=A_ zT9EAP$$s~HW1IRgwsGIe)BgR1RgD4gjnnYe&@QSzGMoDzPsZi}mw3fzTO9440_Un# zxPD*`J?gK^lai&xZ%PMQ(`6L6-8v3C7j+1uw{4`74Hb~v!wDr*Z8-6-JRd&wfOcOR zKzZXg^FiZwiu&luDsg^PJ-1Tye0x%~(#xnGy)}tvO{=Ek8h!C`;7qvwE?TG_aFIeT zbPBt7UXd)HRL=j+EyengQn>5nDV}y=Jh;h5^EtzHX!~(9*iXGC4!rtA(Apm)=qj&d zt$U@+%{#y$*b1E+H%jsrnes!k!5q7~ke*l#LyYk}3cC(%#&DFg-4Tz5Z5G z`WFSrIJ}**{@xN(Z_4BRdc?h4!|q-fx=&XC)zd*K+{;@k;==- zDI@|8e|#WVsWn3ino#~^UwnTni_v6%YOx@^-^4Hy`D2VEoj1H8y-==iO-x=Mn`xKI#wfiLzo$qtmp-6epvGh ziyhQ=oh9#YQXzkj$&_|k7q2KCBI?m0zIj?o{|;zV=C1YCr*97CGp7By>`o?^&eDd7 z4R+kRCxK%RH$mEGHD2x?$ri4_4d06S-rQ>b32D5mpcm*jIq;!3YawM=D$ms}5)?C= zDD>Y}PWbEv>6x}d(BWac;j;!_HPr`8y=ApJ>Ua z3++VNr~SBNrz&HB4*$%IWub?f_@Bxwa_Wla)T8(5`k?h(uQ`m*e%nk2JKuuciBWt} zw^5vQ)&QIBuhHhfK}eSSAiyes6vMmVT@N>YrMM5yEuD(b6t2VOa}!uw(nxC48cAVD zkE)Pamb_wL7pW_q6FqF>dEZBE$=}_ctlMuR9X+v@pZ-;+BKs*gE$XW%>)40lEAGNU zYfTIz2Y&AN6&we?ragNE{Bd<3?kM)h1vZ=5?#*Y~{#cKN2ghji&mUl%T0nuSp}f6w zJQn!=hy7zxsa883u5KKNk9tpr43&E@arbIi_u;Cz>-l4_%sv1eZ&tI4<4jV%@KfW0@g-AAK6b8{*7ec`AV=3&}ZeUWZG&{LP}}x zKvois2aUhM-SKTSZ@Dpt2TJq4JAS;Z&6vFxcS;P84OyS7!} z{KeH^T{V>Sj8kx?lPhKAeS_of0r2p@2SS7NeR$p^E&iLd9;TPPgWmU>Y0tYT^xG9J ztaJYjdjl46s&0u87}tXz{#XZzubOD(hepA4Jks5R(|DWt9e5inC8QNRq;D#Hu=LGB zZeA1*+ohd&*P|Fdt+a(cEn3SR4%_&9WhRxZE`!n`3aGg2jOdnaj}?VW(ZgpU$p$WF zd-yIw%QHdNsXsdh420HGpDD<5JakUT;MFZ7u=>g`FfG%;TlUW=V5%bLrDU{PDpE|3SQv zlLdFT{`@UD25!tws8#6S0wH)R&x#RBx=B$_3NWD&P;@ahF#5szZyliV<9K zW}KKLhIx3yo*po^j50qeaz9ya=X2N@&UD>6@1|0_43JQDm>7_|JzPKdN;|710 zdalM%`{&cR{B$ZU4aAz`{rGL^LjJXZs*Ev{dc$IBOY~jJ{TB~m-F6N(| zCUJtu&^??dU>Bi zRI)IKlvk&-W8Zo~*0M-^IaG#a^7>NxrW1lftTVoAQ^y&8yI7dw4mR7Lz~5ohB(rw+ zfo+9N*br367tPgpjEw?Cnwru}wxhPF0JuN&J4wF%pguwxH{Ty8=#1MBBP+vr<)_)$ zy3rYXeo2Q)*Nss8Y#=y{na-g;Lj?6)S-!H?5H@cv;K6?nb6#W&3^{3z_O~ZO-EjlS zjPPANZc6u^=^4o(R>$bP9p~WjrrQ7>Anu)9GwYk+ACVOR_sY1J>w#Ag%Aq#2gX%l-K_Tbr;w?S%cD3@t{pxr6@9Qp$Atm}UeKJ+H3 zt9%rfyKN8;)i{ysx=Z|Vr&qP{jV?NG90py+BZTDmNg%&Akef%h&~D4^!tLghG(-y8!r7w zqqC0jB8^%amv@*x=>!>@MB)FRD&NraFty0Ovab1lG#bos(N^A z9A6!OPJB9H3>xoy2R6bBx@MXPM~vT6?a>HY)cg(tdS#R76-S$P6~aYT2g&WLHvG?Z zDF-_)q-51NQ1S95PsMVv;KKo2`ml*|$Q- z?F`(GaO16SY|$*ojIE|Qv(?-N_?G0)0peWro^Xu%mR1R*(?aKiUyx>Ey)Zra0T_~m z6(c@C?-Yq(G*X);w2b4MV|GH(zU(Tm=8K@9c8b;nA^S80<9{EHlCNS8?bFI9)$l|* z`1AnDXZ3~FeO`DkrOHUV_>WG$;3bgpD7zYdqhTfBu$vxlHkdj@%oaK_&m zd9*=qF^?ELl=#4CezB|oqUBBa&zUw_Y|tNX{~Ax5PK1i_C-%^P#%A4XNumWs_9?RjTC0#<~rkUa@-?6y7T?uF3a;4n7S5?l{C+W)Ta!S#W@QZ`K zyh&;eZ?cx~V7u8M{b3^i4SWtpSLa~Wx6icVd=hSdFdc(jv$#I_FzuOYB}&~`fMr&8 z;CFr!&(YJxf*}EXw{|y{zbU4;e|2Q@!UJ#mYJf-BE(#8b1BZRd;>SC?V3NEMUeo(a zDd!h+?(zvdSW+q8Z1AK}Xz{(GLv(Ps9d1)frsmLS?mI&crRU^>?bKB` zV9$3ld|oLWjycH>-%VuKkwrM+gTG+fN1b1tde0v|#PeD!8ywwd8BIydrwjgxSibvc zmEGDx4k9afKSLhlm#t>uh!+OqY`%8sGwqtAO2b_wEasdcjhF|V8Ci&?)fsGMkcuDY zOYxWO&g`2vR`eYMc=vP;#q8GOi6@;f>YgfT?7quQOXF$EDjDqaB@)uVoq}ieTD&@Z z2Mr$Vf_7KkDAWPb`1dKaxO$5pPL&7EBbUTmFUo}JTn;*igg=DUNjWK$q_I|46<~ zhc(X%e>TQ3{@uc#dma(?KeFJA*CntbwKvD5_24pD#K0##VM4}WIQAiiD{^Pj?dSob zUGHetjB)0rZ(q@Q6LWsm(iBp7>li9$gi!N`Pg8rfe-(R1}uiZ6y&7u@&pKrj4!(>s_cmvug{e+;K_k|xO zL(y#1Wrr(?5>}}gh?C!+fbUP1a=LK>%V+i)bKNnHolfM_%Z~~Y=iVl$U$&3W4RsXs zH~xnik)_l&Y=Pi8xt(0hkB*Ajycw=+?TJZ$roh&}sggWD7ryA$0lMGqa5)`jIcar# z7c-6zm-NQZ=kjU))ezkLbvC~4NTJ;YcJSXJOYZ+)b#?!bYcaLatLn&t-EcQCmkTT| zNgmf)@L!cJd|}HQm=>jv2uw88HlY5ZZRIi}Zr5H@tE z)2P0{*wE>`BpQIEOTN%FEpBHhOthwV8+dEUE4uTzD5 z%gTz6oyujkTkE)IfCF6lvmPv`MX=FFQ#LbsLh~#GxJKg=ts*n>4t7JInUi?d;}FPe zya?xSO7Nar7nD@Hh~)3aGLIJV?Lo6Ssb>?o4S7h%{vaDx>+uB7ReU312Gu>uK)>3T zl9qCNw769*+)J3kXQsPgyxdZ8opBrJPLk##&jW-DZ-dPfZ9EvZjSBy{;m?jv=ujUD z#VK~soE*$GKl3^6Zz{Z&Y84iZ%i}L5AzU7EfwXrep|6ek-0#C`_8Qwu`5mk;22$qETm2~1w_6H+HF z;0rq@@$SRVp)_9||J|7{^lz8OQ^rr>=C%-gf7XiCg=}SMpHlBG<+1#DcwiDVRl6(QW<$S&cgg**xB1sOX&T12k=UH)U!G2OQSHc-N zdF+;@!ixXx&AawwEW#Y|nmC|9mpfT&=)!Q~ia3rAAO!c$%r74BxoS{5heN9UF^b-#-w(oi?`%<=q|0$z_d>%Wdt5oj zp69CvF|~Md^AR@+$w;7gb*9{_e>rFV(TB2j8~X2D1k2xYfkdStFgU9pyZMbm0cCg; zEyL$@ik*{W(82c=gib#Wd!|{zcbjQ^WKa*>p}UeVZOx$4EAwH&-sODJb27bM|AmeX zOvQdj@`csi_s)BzKELcph4qy!v{0uvA2Qj^>zh+CZ(tE$^sVAtlRRz+kL5A>m+1K8 zD%iY#GVL~TMzx{Fe1G~}QX7+u8Gf!p>id;o?sx?HRJV~&l{F7QPqYucP9ce^*!--7 z2lq3<)+L=N{Zaw3)Gm#}A8&$$=YQ0w z-?dcn{HzN8T`){&y|I=}m6K`k9XV91$>dAtW3Z#h1D)*U`E=PXaw(VQl#@5++15k$%B`P zhxUJg$$QISL+NBzKDQ0_y?Q_+{QL0fCN(;!=mN(oBlx%eeC(}ri#8s82p3#!v8nbH zS(qBJ{>E+aHd}><&v{SR;`FiOQV^H!C=j`85n-A8iHo-6%)@}@K2dB z`1X}GpRhA!`#Yw%UB!Z>4h+V@OCtE%Ssh&S(1smWxC-;!rwWI#y*lFf4(|1H8t30X z#!u~6(dGCQh!-C5YjtARW|d(&OlVE%LCvvA(M zmSjx2_0yv}$!OpjY+iN_4jl@k2Az|V>cbyts*5zL?=dHrT_-?)i3dMh`UY0{9su>< zX{0zt!sBEQ@{61yI8;tf$foi5af%rRYmMRMqZjbKskiCKif&Iv`6F#rjpvnfP6+5Z z8;;Cg%b|0VV5xlf{aa;2>S`mo|M#6xyiS&MgfdWn^N0Grn8F{E+9aEwjfMS&&#C>x zOx&ksO*L{hAIbPVUZDfXr_~C%{ZZV$4#;2&U;w+Uo$Onnt(H2 zOX0TJ$LXt7AN2d4%9FB!sl+Bll=U=6lQJ31IQtYXNP4r!lVV!@Pa6MIvY=IRvi#LE z1d>zR$kg&WeOjUg8!Vi7f_}Dm=vo^EUmJjS-^cMR6J_{OtHK2(&mq&amWCKW?Jf6G%yjpjS15dBp54UX)Q(HRVy;KRU zpXHC<7Y*31RfRo`mb0W(fxl1I$1|Tw_}W-o+W9A$r%qJop<1Kp%)%PTx0c0M>e09) zDnoofxe|OX$5w5vBM8q-#*u9n{5stghw1b}R|Y;f=N&{=ew5TyXycrT+rW?o>i7$i zeIL{~-B*Dh`9|@}?^od0^uByHx)5hXdGnVt6*QZ>j1`i7`Kiq`8oJ4cMi||Nm^(WJ zwV&;v6s0Q8u{CAYrrFTdX~K_34d!T_bZ#w-gcnaP(x<8{x;nLmt{0c_8^a?YyE~pH zu0Kag1BfyY%Rx-BJ}Pgq#)ZCGJV@G@>s`k~$F@lvn%0{?-MoKMXQZrIz=?x|lAy>uD@>WczXXGApoOpw6Eh-Fd$1BO&Vf zG4Z8KI=fDn;i9u2sJW(+mK6JmBc^T^da0%Gw(IW%+2;g32ONVCgKM-fybsuKJ3|JS z`|x!8S$JQoJ3ge%=9@lasM0SL9!DsnSF0n-p8g@4y2Qaci^1%W?JO>spu;*pVtIq5 z6jjwn3#}%?_4=Qfm#~vP_PhX|r;QnIWz`>~NVLx=fP(MG zq37RL`uxa|k4i^!omvKgZ$2;1w&XO`jXe5U5`Qe!L|G>-jCWDQm(tHc^F}ibZpeeE zhsUW{#(+9vkBsiGCQn!1yb||SXs}#(FC4ifh`R~r=g?Nz@?PM7T|sQbP9els|BIRX4s zm8o=H8G+suH0_~Io#9n9*581;YRmbc+A&v^h)zDo%J~d5g`S1 za^ho9nD~n>J8hu#$L#rnUnF>k2ZK#%Dh11_!=uz`xF>xA4)3ah#uJtta=VUu=N_bD z_XBYBrIy4t>mKd@zLc*Uo}ug8YQ>Rn$5PBEZLwWb0jgFipq(m0?}>LoVQxOVOml$o z>7l6Lc#^-rP9nV(RaC7o1dm<%LW53qk>fu2-S6^QNG@xjrWS=g3bjPm|tHTb`ZL-KWU`&b+OIy)xAxvitv+zY68^ zm6srNX-@UFZmlZv#6D<$P$yW}%%Sb^wix8UiAp?`=|}$*!Tw(eQ+Iur>L82i2O8*$ zy%P(8v#KYbY8vH!=#;Q_@e~@O(tUmxMPU2%0{ZD&1LrihKtg0Dkbh5f8aM!xuB)Qd z*;jC+?jen@+eFeYRdM%G4dCaG=+E<7@=@2~^6+os<)eDMNCI@$H=S#pdx?v~UWh*9 z!_jWTWN0_ktbX!v51sgX3@3X;;@vg-*z4zY8e-f5W*)5|XEl+{Ce8%CfHDk?YN+;_ z8OQSj1Q_r&9kY{%km4$H?t4LF|A9PjqZCG&NV!!GmuTwWWYQgIoHVy z*4@~OPwph+_09e`z;TnP-lIRA|DGw{y_HTgzUSfa5g@IP*>UdpCJdc%wb{c*Xc zF6V8{;E(ruu~*M45Zy}_Ew7i+pEw&n{cj(YA9@V((rt7(vs4ViVjjB84D~O&;e@Up z{C&|3KBziJSP(J_eTxr6RmW>Oe1ANy>Gt|}T-!rab}Qh+CxcMqv^_WH-XcVOXPd%S8l2IV$*<5B1Cnsn(7Hi)@K zrc;m5)LR4K_@z`1PnjZiRQ(k$91P-isb`|YLSw#qw-_$HU&>Ay8nkWq4>57KF^>%l z=HdbAFfexpS&K?MI{Y=@2YuLM9ShQ1^tsRLGERu9f(;W#qgBQ_oKU6%IUGti{ruT) zrWMrHHi|zscfz@V(OB4K$lgnSi8Jr$!#s^~_)iRHw>^Dm-#2a6G$`ZA@02)UyDBaJ zv4_-MKf|Pe)6ls(o?Bis>3{xBD@*ReoSKv3maRu5kB3TQSWAK=ZoqghKa@t(mEO|t zy>1*lP^tQXi8rf}VbtE4l04d0j$48Mz42`f$zD;Q1Yn`57livLvJ`}rYkneYzg z$QBR!60?%^>iTk0ts-9Y&ElOGrC@xm9ewb+PWHARXf4+ZmO8QAxTi?8OkTieoOEGq z`2%srqYUoxP?fbO-V*Ou?-%VKeWWw0J#hUdc^p2W50~wb62{2KQF_&CVP<_IWK_v< z{eNE7j~DiU+xM4Y!Dp`aTUal29b+m?T?CV>wm@XP3lHig@U?B{Mfc6JWF@?VXYE_5 zdiOM5Xpt>^h?~cQmPc?vRRSDnGT*`^17HF0LAF+)H*`Y6S`h~})0c(Rx^kv=WE0)so>ic3?X@{_^KRd(>#wGDUu& zzMrJ~zYrS621_2+TS}a#o8afPo)3(q{Ahl>JtvF;u z8MQ&8j3rYns4ubxRG zG=0(0S%XaKcJiF|M^xc@9vi(%KtnqhtHz$d`-38cBy%@b%y8$72R^vA8?A-0>xXqw7|#CuKC>_1(2&s6xZ$|wk0)$-qQPT&`E!=qZ!u^6FcuHGzy;IaP`j)L zc2tK`gp?Y5p0ST6FOwICesbiHwZnO>^m7_EKm#3O+_)k!mT#r}qH{Zg#Dq1O+~m+I z8B_I^?4@s1&(H`LuD{D=jpg6Lb4o5<(<~tEsD3o)X$4Q{r7MoVtIHnit;CmGM`Fy8 zFpB%_$LD%Wlib@MB%`IxgBGOVhvolZ5X#^QGv}($F*4lYxS02E`VU&JB=K^uIAQEy ze=$p=L%ie>j%i!F{gtF(4s9{PA%C}%eCR2hP(KrXMRwN|3l?MDtXLd0YAHI)|0Rc& zU6dK-aJ{_M{y@xA>_7(dOH z{WoRPSVMmplA zwOhxp4yj}0NogKy^i4#?FnZf1g`ZN6kiUTycmDnb7Or2#KD&~6NzZPkL~Vl*x~Wd| zdA*s37@MJ`w=oRY(qLzgT5<6cD`>2lhpP)q#Mkap?B)CeGNXp`hpIj-Ie8U+^{6DS zIm0mzrcuez0DReyNt?9RP=rv(i`3I;i_%PX`R0WW7x%_*(QTqax3>PaKm*NvHuJN- z`y|uXSi^T8C-5)p_F2cpaNy355Zro!v%bjUx;ax};m>-BW>QaFQk5w-H=bt)`LE*U zk7Lncs3QJoR7IUU7Ysd;$_kG^h@DESxM|8K@ynqRd@FD#Z8ef%!@7A;5x)#SC57^x zcX^QIKAVq43}nN~<7B(7fkrQ0$sQq|aIu>^8)2&p8hJx;N5&kyp??p;?Z>jmqcm{z zNrhI?2v%9SvQ_wOu4%C5q~d;@b@w=(FdE4>y8H4OZT;A9KqRZw$aJ$tu^=dSitBdi zK+lO$teY!~zU$;s!TF)E_Earp>B;l-`K#G*+)l3UaA3`EEtDY}1cPD~xPAWyNUzM} zxwZe%%!ZX*+s#W~mTQ9+WmQ&KtppbKuJB}cf4Z$!OFO5_;AXLizrMQ$bDHh&=Fd|! z>BAo?Q`LdxE!G@AH7d-AX=M8rrz;n<4=$Fn#Q2Y7<>aD_A zh&4t7we9>{W(W^Ig>k&H;3wGn z`(oVeW1`&L5tQ;^F^49-B=v@0wDfI9^;4Yyx;vqYa(?S~b1b8I*D7`LKi3EKwHBh> zhai3*D=$`U52oYO$8ycYpRgcqCRa`ihndA2;bQh(y7F)|#2Qt>*O9Y?<}^jxf9WEW z`xc2UUp;xT?G0)vl*Y(8*I`!1PVq{1HCzwALY~tv(aP3Y5L48bn%=AM6W3$pruhR3 z$|Y>R(TdLo#&Exr;jn$EHNL$N%VncZf!BvH4p64Ce2DT&9iq3lsRkiR4yqK4QJxWFzyeNZQGB08Paz;oC2*n5fzJ}hj2@>7Yh zGi)I?oQ)BqzvjSR%e|CP;KGezPhr7T559N&9i%5$Lc?KQ{@AR{TbBDl@SYNmGSOs{ zoVOHu@g|wf*bl8aHe~U+LhPw_1lkkTQPHo3w){2&)0~Ag>vtC^H~m1{Qzn9D=0L$^ znk5gh2;pzV*I}IVCybVFqzZ>r81rf;t82VK`5+ZG78k+R+OeFHo6Ii%_VBrzciAhb z1?+v~`JPUo5cnn!+{Q+8T!#X07+ixnnnq|-_eD$%Z>4m*bT0c6!!5yvJU2~QsMU+Z zpLM!o?wZ~lDXS{D<0xKxqJrD96>;M3hN^G_V}2U+jdFLL;g!Ej#26`GygYv?anP9Mhloe5aHupD>vk{$DH+Be#a_`>rH|p>1FF7~dJzoI2~fT- z2(Eo809S)bc%r>Z=#h09v{nZ}uO5wJ--YJz&b|YjOfcKt;ZR>plk_$?LSh?=p4m4V|DrG%uuM83dD7uYuIOqIX{su zWuG};=;878=vmu9GPw!R^(+N+GH%f57wSAcVh64}*i4#!azKx!@uF+F)boKh6#Pu! zMzNV(OG;P*J#d9ZEo}ci3GS5~C+n_vVs))8ZmG24lh2olaX(V=a()%el34IkR}XU}Y9zBQL zN7t%7u4Ull;?-Q<>4~c@9mfv4air|`7>XB%@x@ICAUVi}TlN8_bnBF%21OEAvs`?V z{0|h@pAZVnGVwu@9jDGnp&_#}aO3Q|_EF;~%#$<5e^WogZy8x=i8O&8!aCf)%>g^i<@tA95Bkz?CY}kO#5P9j zutFuVn9bcxQKz_2yoz z7Z%Ah=eoE-b`~WCqshnmJytuIL8uisRMTZ;x*igxv zN9tb$SZK&!XVjpaUV^=IMHIid`lzb?#1q`BybH=*47lFsIo|ZK;u{XbaD4j|^c^{y zgUY(Uw``K+>Z>Z$Y5X95+|ccdD){3@^WL~RVFj+qP2=3ft2ss46zvX#Lvp7pJ5S9) z=i&wM*LMpS6{lj4j(EJf%2k~8qKM*qeIz{oRP+WV3<=MK<%4?i+{4~%alQyM+o~w~ zJ z7N0Aj&E3zJ%I{I~{T;AUt^|5NE`#L-53p%Q9EWb(h}Q>9LoMrGn156j4UaqGWHWgV zyDX2Pl2urkm<5Kvmc!bkGlZd&vZ-@-4e|k3Y+Cw(Y&(|Ix_MXesF;g)j% z3uJ7Jsa&yz=JnhIeQwl=tNJY>B{?_R9=#aN<^1qKOewwB)W<9G-Lrv*5+GM2fO4A6 z(XHDly*j@fG97%#I{O6-TeX=S26e*oK1ZqiU@^K!mSR@>dVIZoF(y^nA_RxvtCUP= zyThz_rwFSrZsuk;bE;E#j}OBSfq%wvyqi)lD4UHJjaEBzTR|;q$E%L1eci3If4qT) zkVuoCoy7^J(ZVsgV0KB)<8?^}0(ZxlkNa-Y@cy4@$JJu=iFLv=arJ`x#8OZKZ623* z6%y|%^6QlUAmczj-1oM`nDbh=DR~6CHajcc># z<6C%v)<2}FJZ%A-zc7x=(rn?R&tM#(#NyN~B{VHR6n&;nM$;O59%R9AIrkUd4!?pE zcH4@-OFdCXvzfBmR-@wk7%?qe6>X|%qoPPsY&?sQYkce z?k3snl{_$XC!RU%jrl2dm}(FS8x)t|>&wHT?1uq-nb9b!q-~&lncMiR=^B9Xc3gH= z9)j%~=>70Oe7MGm*C{@L>9Z$eW>R0_{_-nes}aN*7v|&FM*UtF%Dpmnc4Y(F5~; zGbB0B!*>(MVYJ^U-ap-+hArHKC-&Ln5vyh3CvL@UrgN}hQzf2OoP>rAE@0jA06O#S zaLCFBP}pUI?Quz1Il@wM_S;Q(IVcUUEmEc-b}2MrL5yJG^o;cWxZ}KvV_2cz0>4t{ z;o*VHahSsqVeCQ&tn+#VnUdRLcBCCn*S(%^C?&8yYC$yxwN0IJY9g|hUxgxUIn+;rDIx866Sxt zK{cod&%?LEHccyxXwe|6Ws`8mg8?X9TEuftBywboHFj*gB5_vfo=40-C7f9l$pzo4 zX;bMiUa%Lt`N+>C-#rWZ`D7UeG)9S<@AuP&f>;>y{TYss8Obk)4aeQz zeBl1nL>yF=iaXEiI?U;@6siv{#{T4jFVAd2ozBD5HMbJu5==3&{T!^Gu@Sch?!@w` zC$J`@L)cjzgYO;2@xOz!F~~cI4D+n{&(SMl-tS=8yV(iTDqq8#{P&ReXdX_h`zr3( zk%6m|Wk>t^baOJZdUH@`9SWMZs31KY$MjL=i2pLsd0GNp>l+Unv&Z7veOGbms9!X# zOW;VWGuZU(B&-f8DF+K$Ao;eU-cE*W)&eJ*dOb%SL zhxg1IK)YZ6>SjPQ`PZ|RxH=>qw;HfGdhP;zb54`vJnn$?w-bWNFMI6$stKIF&ldF# zW>Ct=c2c3|a+U)9P1Mk$*nfg_5>q;4)n`$6N&8mXw z^Mg=D{RB8mS+Yr|0S{PyS?KwA0Y=?W!GOyfcwRI`|NQvdz zs@-DT*2frWo&odb%Che2>F`Fo4E~cO9*Oqj$S!q#U>MpVk1j3b9A`KLaRm zrXfD;)*&a@W?=F8D|qY0FWMEf9+sM4qfu8h@t{q*u(i*5?EC5hnw>4A8+Z1gK~FOn z=rL39%hZN{>kRSb8!Oh{Z~}tNPU5eCT9~8QMQd#WG1=>hAb1}Y#lMr-T6#0Cf77k+ zY<1+M3u?F%d*Z#Az0~^a2N}=bh({p@KTb+QT-*~6%IRTnbTTTb+2ho(0+NzBD>$3P zh*FYf`d(QiGGq4*C#GE%n5R@;43u_$Mm3;wF!rftSx3u%?8f8&o zjRdV9KvIt1ipBkxnQNjJ*AjAqhCCXX?YS#_i*G(!Hyg3s1RMNiRm_49uOeBcW6a|6 zRkIH%l9)es9&;P5Vqbf$@UVpy-E|mCMp1-5KiA`%n|oOQ=1I6$;AMsDoQ3eix7dx9 zPS`R-igj&01px;$@cpwRQt9S88tR#bP85i~tAsn#wjf-Y z4R&2#*g5KfK9l;GUuOW`UZcYwPqxKzVgA6kdy#>07Te`_jw{?ZmZIM6VkV~2_)lyG zbb}-A2%k?`K4Wm)cV`TlrpaF3kmn5zSK`*Bt1MJC2X*WWal}`Xu0Gt!7Hk+tU1!Cq z>w7+P{uKhgc2A(pz>=lE^TM#&Wmsfh57S-L@I>Walsc~{$l8Rl7Xq>Nv=`H!WzBsW z31@Cyr}0q9evEti2z9$J!R>5)4E-Jp7w*32n>1fS>Be>lcbSK|aia9hyOTY-35_{7ypOU4rk*BGD*h9&^oA#%~)gLTyqry40OxMq@-# zBW(#=SX|AN_ML@mn?jjZ&=~w{dI*0URKrZJpLH#%!AlEe@LX&qL`Y4>)Wv1UCO*cc zyQLWaa;0ELb%*htj?io1j{$!+v6KJ#(4c7=1|}f>sE|dGg&{1lu@$%8jAbf*Q5ZE| zk(vvIX~L_^AbMFJx%PCHWnaj|#tGBdGclN{m4nib-XIql4x#cFu-{&RX_*zGcKs#T zX{}Km_WB_HEVsg5Wks_gwVBk}QvfRc8O%O2jZbiK#=YhLVBFMMc<8ANKI2mGxMUQz zH0?r(`Z#Tay<+JPNfo>=^2G3DKw%ucPbHQN^?i_1qPaP@#8)<3qS^wPOZtV9;2 zha~Y(v^ULLEJ@kPZD{sY2-7AWk!~b5-#LnkyQCPv6jNHUockda@ky4;#5q0dF zeJNj{(8sQ8UP6DfacoBYQMj?A5!bn9v&?g1WYJ}e6@qex;D-sA?U{mYv%kYbLB{uO z`4se%vq$6bQ11H|O`L7rj>abeb6Sn??g=X@HQ#6IxZ^MmEO5|^iq8TezfkZ)*i5i@RoF!=V^t*9nUo zEnEQ^fy(ydc3O8|n zmi0C>w~pyF%R7%*{H#TJzw7M%8dtofrp@=N=fK;4BbnQ<0c%Xw011s85bnK%%JLzs zu=Q`{B`tRhODaXxWL2_pI7CqkJ*m6z62@EfLxaT(3N~5GUTNB~j@Ti#e?_Eem7)%| zR4-vU^%1bxZUXuUT!;PFWU(rJ9C?puldi@?RxZFxhr>B4GJFWrB&OnUK{R~2V~H|> zBG@0g6aROmo?BueLqngfah$9I$!@$3p3%{G%+3*8>of5D>)EI#|CYD)W#AVmhdK{d z(-%l&HNT?ie%XH7lrK_U`M0X-*Lr8%t-lBZzN=!N4u@yoMDx>|7vkbnTjE~a=4&>- zsb&&>7#g99r_}*^;vMO~fBIQ%*&Li5Ax=&AOEGr!QCzp&ov9m&u>GG>m_p}c7F*NI zi_aIuZR#@g!e&0`Mn-_~3>jP{z7Zey`eRtj7ql~uWJYUB@P>Q@djFb?eX75?|F~F; zpU?~{f3mS`)i+S^JC2%agm3E4aQAjNVTZprTW>j)mmWHSS*iu>$;ZWHd!ZVg>WXn` zvIw?pie?^NIlQc%97>dmldNed%eyd_%^jPGgUi$DLSrh}McE;&h@u-S9N2gDVNmww zz(LU#+qbuH@H>KYvWTMT{a2yKW)6K3WU)z6dDOSiHwgu%fmrf}h zut>yfPp*=7(ly#%E02+VkFaB=BNphLr@{$!7#kG})1TyE#@!B32yvjUh2ALRF_GSd zh+=_e5UAz~IMREA!2FzW#kY^Fj#on&KNBW5W+tW>Ya$o5iF0qW0=1npu+BXOfsU{@L+Mi5mVd1JbLl@+fiwBl3;h&qsDP3(PW%8Bzrrf z@tg?kC(f_0#w};~b6@TjIGps<3SRI+Xl+}}1Eb;^E&f`F?^nEn%Q zj45Wm!76BND9DQpY*^oOIZWG`iWk$9Fw|Kemvt9oX2MDgn=hDU-%8*sBQJp;uMTNO zY51`>5(h$*$z$aa%55sgjF%I!?{^ty1TDrsjVKt~umx<*wYUxH4eXEW1>Ab9hTF2n z2|r4&#@^fg?3`6E+g#~{hlCc9>SArKJ;sOg{`Lb_)Lmv(c}iq@olLd$$WziDj)fZIG__e+Y(0Y?SKNR)LT0?^=|HnrR)SpWhaO~al|spqL}qf@ z2W@oyaqWW?=yv|f8IK?4b4CH&-V~tDQVW!_JH)HxGCsvk1Qkow(WmY%lZ`8dCe{Hq zfuZ2ORule>7R(3!+Gr8e##{y4Y=`|S%oENzCVQAHy1U?QsJR)!B8mt%!~2~1R&D`43bSU}SC>h0@;P&O=y+($L2 z_Sz=$d^8=G>mR1vJ2wQo#RC{nu7Jtgb`wA41bx_FLcT6ixKk*OVtfQnCm+VcihTP1 z^b;E_Ttf##PmeX^QY4_DJPAYn+wHwr5PijW3e9`xEXmHwSNcCZR!ly{^%(LP75h z6QS-i0mR2E;57SI3U2YI?*~gMWT7%8%S)22tT>flp2Zrz9VWBn11!fkh6c_|qhnk0 zK*&0YeO?lS#@iEV;k-cFus|4VgY{YR!w7aN!jWY2y_oGwTeicy1V#1Y7`J#1mVNdm zy@2=h_ym!NQxv-xzkbt&%GIk;X~zykZPW8%Y&%r2poYvDamVreAmE6L#h{-%)FqDy@H z>s}E4RY

$G}m~7mo+I;nGVYxXUC#U?n)9!}77@@2rd|(vCE_#EcCVg|iK1WvuRP z8JKu`;e}lj;kkG$V6rv4G(QZ7vjrU5kuzLR{~l6Di2L9M2zn%+~xeW~K-1 zSge5_20fQS^Qn{2;>2!Dk~zy-#msQ-%p2^j&stQ!7J`pPT@e+ZnavJR#y?GV^iR4d zVai3k7abTF9mGe+&*OvfT6AWqI=y!@LDwnvNbk#FN0tz?8@z#GN@Gzw z*#xcX=it}K5NK=oWaed|#aXUh2SeMeFjmKxLhi&d?e-EBs{X^*4*J5f&TE|W**4sp zYCuI1bJ1A;D9)ejN|u&L8#gPFw_hCFKB*h5q7?APl15hhN$}4JnE5r)x-{&y55~J@ z;DaB=xLrHVOfK(@iOC%m>_52!4f>tIVp9x@-*6f<8>(1uqrgx2?!ue&c0j_)sQ3G+drcA;{0=0*?(+BrhGB6Uyro8JEkYQc* ze9nI;DI7)l>W|=s(MkIGpDj2i+fZU{Ha_1aZa&dB0II!K@l$-usmNOtuUZQ{L)j-x zDNmOye!YQ2y}uB-Kb>W(2{JOZU${l8-pt9b8RAXasLn-+nu`0;_lY?>UHXWf3W~=b zl@oa7i!O=$FNDh@t ziynf0;!_mL6v57wt{C*VmPS+xq0}plO#c08=Dh3_%UYkrHRsQ#dpTkxT&BeOxd_^p zRmu4cMzO8aGU>oiJC-^AGdR4NNAF&QvD(%vtktrXMvEe;!YGg_9`j-rQ*MI7!97)n zUnVmB#Ra@hv>$0dJ3g!XTJ|yVBA4zid}b$tzYVasplPVO0YZ4S1aIZ zT*pxF@%!w{YEvBf5ss-N5(2WfkNL(Jq0H}RutQUi@`49h^SzxQ?Hz(`M&F?4Pc3VX zdJkDO1RpO3V@B`-5>uQ-(+hRLno~yI8MDw%cPa(>PQ$Xdp_nptE436%Bro->N(Il8 z%sW7XUq03w?}}GIWp6S1TNi+c(o2{arjN?^1vBj5@$7;sR`B{c7y!qXP9U9dAzCWUwztdGknjVN5-G`a})9^*)z8+F50b?+1eC=P3&A0 z)0&P>qW!%c|T?Ey>oO`;_s zSDEpcW7N~5h1FUG%GUP|GX5^yf(yL!bHOtz_Q5RY(ZB5A12x)b+Xnv! D(%2RG literal 0 HcmV?d00001 diff --git a/demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.json b/demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.json new file mode 100644 index 0000000..77f3fe7 --- /dev/null +++ b/demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,50],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-traffic.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[50,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.weights.bin b/demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.weights.bin new file mode 100644 index 0000000000000000000000000000000000000000..a5ea77486f2cf6fd00605c78d52f32cd044ba8a6 GIT binary patch literal 22276 zcmd>__dl24AI9yKA|oW5B$Skp_j#S7C}|-nNkg>sNmE6($R;7PBFZY!;C)`Fq)1Yd zRFbrZcG~6Z|M>oV|8U=r`*Ftgx~}JC_~rzwrM7gNPdlCLcIlz=U{?uc2|lVH7tENrStushNm z0Q%yG$oQb^WM6UxDVlp2&yf3=TrZ2OrH8SnQ+wI$U+?gI<2n%4R|P2Z#mR{)ATZei zBH~4HR8Iq@*C=wk`K_eo_1S69Y{t`O!(`~6(~VCH!r;Xu5EiaW#_`o`)vo#|nnVxeGMEh%{GLTf6F;C9ed7T{9H>@}rn zkf|@~v>|3MHX@eUAMou;DIRN}#b%y80awdrf$-Ti*rQla%+J08KT~5!G@Xh4_N{nf znFxHmQwUWSn{koDEq2o505fp3=2t?0un}sNc*Wiu@A+;9W;6razA$*{)PbIU-*Dl7 z+N^1H2fQ@N#=ce^EGW7RQGHMIZHA z(;lg6XsSxWGwc+4$$x1bln-=%cJAq4Hltvf~+J8V$$cO2#VqJ0k+*^RB^xpkR!RI1gcplpZV{ znCACsJ>8(Lz^|A-1@WQvcdo_qYkw|{BAake)4yN`(c;J&zNQSujw+o65s=`UB^=Ajpc$n zdnLGF`(c4~{#_(>66hR!4UVQ2!kG5Cbkg2v?A_o&mS)YQwMTTYW~&?5>0)?7W)67U zzh)E1Ux5j$I>Ej90Za-0BoK?30GTJA!$a}6V4pt>mu4GLzjqpRFi@Lzp3~(TuJ_pM z7y01(C=PBCbE0$#}nerD_ub}LAvJ{u#Qgq2XhS5j&f!XRyBz_#n z^%>b@*VTtG&+rhrd|!=^YnW{qFg8G=Xl_``|6@&6V5h}f@n4IEcA^1=%?9g~le4>xC)ZW7ke>8yS zl>in#b{)5~GlQ7dH(|5t2zVA02kK+yVDN@T;5+p&6kRsPHlIB#$gWQy@qHWwj0%FA ze;46sOCely*@e@l*T9>k$@F-m06?q?gL4&Wf~h!JH|!-EX-I&czd5s)O~E4ZZuaB3 zKe!I7Mt}9KaK-fuax~v^G_T0f1@j>RZO$nbKK8My9Kat;K?CiwM zr{feQDh!*w66`PA*m-~G#)gyQsLgsAyL;M}MD5F0L2FhkUQ#>>ChtG7P2$7&ezPi= zsh-PfOx@8s_#|Y!lcn9_dC)##G_RPXBv=|2%(|>jlZp8i@bKYE2ne4Cs)> zbN4VTpSvF#RpsFDF+#%*??SO3GW3SV58-j0d<@)XNvrq#gl`XqQKisF@NU*#EJ(@4 zNsn@He&T$5+LX-WPnqM(p7ZGPRt2}DWD7odI@0{G6mC?rt2ia z7H9Z&Gy|ugHr~1V3s#Mq3}F>j*jyk&ZERoT<0nR_?;}GKuWZIIWtBuGshFH;O9fd` zF>cdv1-Bh3W=UrUSl_~vg6~IqapWi+zW90sTiHC8$~(Bw_s;jg{iGiLO0Hy0wT#`XT|p>-%6*{}l2dkziWf zQKo(M2We}$AoNsI7d|L26^yq#i@mEC6NSTP*te$I`n!4a;p&!o%sVp&OsfsR>e(Yf z`hiYxI`NWiZ#<6^oF#a}wG0?89m9UP6)}k&V*JKMW$>RMfG-)J?H);hT{kAz`zDaJ=nT za?x5E6tm5_oBU%^nY@f0v?<2et;gY_R599rc0j%EH1K8*Fmin}S#eJZ#7}GD-`%yi zXQd*&C*p`&drQEf*9NzakcQ#=j`7vOyv59ymVAt;89%+|6e^#}5V|k21zn9O95%2OY7;qzj@k(|MVUCu zR*w4{`U-4AvQTq(APrKmr4`l9P*t6Ug)-%Wo9-Xr;?7rCwo#WQy&Fr8iLapY%PP@J zIT-#O3Zd0If&Z8GgE*(vW8M8P*lT1CH|-_(rnnDaoGri}YjKpy*uwT79f2=*X5mcz z`LN^rM$+rKh88+a!Kz7J?C7_RM02Gf?h*TmjR|F7RWS&#L_n^QO0p*Ik0br$x0Uj;LAezLEk`*E|%MBcYm zk`|nAWxE4C*~&Hp2x_yUC(bpnJq?*)eMJIR*XszcZHXtTznswAI3I$gQgHM>DH3Co zio5r>5~+w&Y{ty*c=zQ~=5_J|M$QtDuQLS#G@XTCW7a{uO)txSkc)XqOVE0BK3QuW z%@)p|%K9bc=*OFxB*jP(_hzVKReV0`ztrW=J&p=BqXnSVy9cL)OJdr+RvfNDG32cd zv=8QzD0KnL9aGEhXuXHzkmu+icx34mY)S# z-vX>vI12HK@px^5E3}u5x06p&;R!d+z|^c2)a&dbk+CRbLw=bO} zw+3&D6f&jqw=k*i2FuIzM34Jt1s{xE*&?GaFg@}aywCcA8Pg669i!su>H=+)aMYtC zunw~PeQ}h?16Uq50;SIEz_XR}V9OL~Xq#+LH|7pvrIHq}`n;P_OEJDKTZDhhy@Ey3 z7I0#z16K?@2NOphKwaq$cE>aZHTEoppFef!sf8oaJ^wZC!BqHE;Xum_t%av_b;;Va zmH6$K5x41D&uwZqL#F-@K5PLGr z;LO+S0;}RZbo>ufzI~1qRrWID_7_ayhkhkl=xa|GXM50=hEaTZ{7QCq#{ts2A(rg6 zUngi|N6}+;oS<%OtFY|gD)_SJH9PItg`TB5Q0Jx+H92vJX=NTG3*(|-+4Hxs+d-SL zbH~Z8_8<%c%H{U7Bt(tk`ehK8I5ZnfImG%syfNIhiZO+I zSZZK`?%}5}EOZOnj}fI`d_x4uWzk^1;4Dn-Qo}$09>a6TeW3iHLn!(tLr^AnfgUH=xiK!ryCXzPeFx95gOYPjmk^Dv5@e5cB*Yi z`z46b@+nmid+?#)*PUtf?BgB~d!~Ryw+vF{(*zb3X)LEG0s}@YX9Y)(lkh1Y;jY{y zcB}RUgl*Ra(WeSj24!jau$6fG*&DdiD};)G7~=Ic4t6bhEx0g#H`srd=gti%@+$s%t~&LX5`&mmtFh$OAldhK z716)%0jno`z?3DkSo0EB*l75goae9LyRr%&dw47h+218xe9J>@e0QTNJ`2vkvetT{oBx4nr=Nx&jn=p*--9hF zG`E|&X%Uqfcm@6$Pw|gRCHp|}BBmzWQWLo?xL?hgycn2_&YkbcmVZKcliSS3XY9w0ANKU^ocqXhM)19b zHBe?i+_f0?9huE9E;@;&vzy_|`CgJJ^$)7PR>6DYX7;;$6t12S zPZw-D2#a=^(AjG(_{jUuQCT98=-RbonqD@BM+rDR_<+fFY2&8Xs`ZYd!|+aP2y>AZ z1KDsjs;bmV)MxGhoej@m&}ss8NGuj!zu-eg${FF~tdU$k?>x*a8G+{vr{UG%hV|CN z_OV+7wXF3QX5v_*N+#?4FLfS(ecDGpBBms0iMiaRE!THL+~s4{QlpgXU^>Fgjh77x?W$ z_v$uWIPZe+%dt`@*pZIfgLm-2+XYZH+zrjnwBxD&J~02IHn`95GFa8LgX?)|EEG+{ zQqy>_`)?gt*q;Xuwt1NNT!RQXyURVuoT!?MqV zL!Jb@pFWK`yqh@WHQW;Bi(SGeKYL)~<5$GvLIT9i8_DB4?dgA`+|l*WEaK892gc7I zG5?(hn3rfNS#|$8yP@F@4-!k*IpxWa8C8#``VXD*wf7`UP3*-iA35GX zx+V&He*eL?MUs47d;^}i)xnmxrNELNKe%PI2mU$u6RogiKu`XMKR*{?)ZP;Ea?%Z9 z$K(a<;3baJ8ZI-_)^g%2GZky)#HnGJ0h8<55A38AjQMaGYD>4kv9qU1m5D1n3Ckc0 zPdr1(2xn@myq*}gj3$-)8%Tovby(nOL~CkK;BL}{8&1B5167$AHbYSgZqAPyYj3cc-v#ue7n}Q71e9*#~s} zj|mKq)(SLx@-gSI1}m|IQ%5x$L*ZmNWly94Dj`w)!;8VhVKYy!=Ho7G_|Y|1Fv{uTI)j)jGsrB z9ykea7s=5eyED*iYJz`Pso{^;VpP<1Kg?G%2dmN-*ea=n5)JYK9g$v`@VHvIMW&F* z20Rn|xv!3iW{+`h$VPJast#3L-YU46wG@Mg=DX0o`$YB1Lu{)+IH6t&f3|)hZpSJB z5}j~D(H$oGW)e1RxepO4BK+VNb6)Hi0mp4qu}i!Hx3dEH>@-|B;$SzfE&dAyDO2g9 z9a*r#W;xkjwvrB1o*=D_W2uU}Asl-vAv_q=2J`>h3oO^!KGYFX&wA*u!Q+~ z%T|)HysLB_IePg7+#b3=e(WdE>UoeU*&M_9xnVd@ZVXH?-N#fGOL1eFK`1)*iR4;2 zU^W%9Z>#Fz?CCVMr6@8V+K%V-$kp2-yDXuF!wyI#_t*D^_cMgiTW3tmj)L+4?R8Btm0h zT=+0Nfum`y%o}p)+F6Vaj{x!4DbVtDKCXE)9<5ud@c8R}81%OW?|$yL6MmJ!XWD1k z2jUKAKDcA}JXM;PSI=TwWVuJ13@BFT;15;5x}Jt&535}i(Jtg8vb}F$v8g^4gc$O$O<8PX zrvv>Ibpf4hcjB#3%CA&bfs*ce-a30W4ca!H{ySof6U|%kZqgg!mjoGXkCEYB8PPb^ z>kbADljGg%2e50F9Zp?654Lshg6p^LKyq+_plW9~_+1F6|8!~uUe6=&O9=3o^J@H& zcC3^{$=-#i>T9yDuPAp&peT(7P{ZJTYeTAI4IWTSE5(9MIq)dCRY=--HZBe0C zkDk_(;k9PLa5rEfc5Qo)CMu<{YPBS8*6Dz;U41ysbR4x`u1(fTY2v4=)*ut+120RA zptAi9E~-3-IrAJa{%b#E9F2$YjBJpqoCdv5oG^EmEylZ~!N;I<=%dSd*gYpk2E^#Z zr8>lH{UIn04S~MV*{Cx8E+}s*#YGEbiS2h&>iyv?46{HSC6nj(sfkm;8|}WzXbi2)(Sh~HR$eMeO@eTO#R(= zu@#Ryp~^Lfp5Ew1eZJg>A68dz(*vE2NU& zz3B!w8NPB`0?58_=5;CC_{u~5L-|M-(8X${}o8k2(CXgf38?7PIvUx)K~Nl z=d$y~utU2;P^J(GTfc8Z`|AT_{jxV0aBVw|y5vLy7Bv9d{0Q@&y8@aYwW_6bU6D?l5P9? z0}X7>5b1B6jCEf^Txu@hh%rtmvE&bkq*>wPl<^p_VK^MS$gsKWHHrQWVWg;4i zXj6TV`F_7IuwP#YTbe6bCi}eE%!m;&U3>2y(gHr`VC=w;Bc-|5CNAf{)5Mp zE|REkRU~-U&}=JKiwCWq!1lHmB=uQ8JFj1Yh9)~P!S^dZF1m*9lQY;<-)NjaheAO;5k&9M8R59~fB~jXbQeZk* zXm@E<02ce*Bz?81?CiE`Jf&_<+7*p3_op$RFzgfl`O%1v`l=vx)@#AP!5M=1?qz&N zzXsob=_AC2ZNZqX>umPLf2{N87FzS)W%A=%0&zD+maz0Fk^w_O=P4^eudz6-T6z-S zspkn@1O2fgcr-5zRpDE0MEEJAb5Nw!B)C*_4Lkp?$7O6O)b0Jq&Yso8%b9V|b8#N_ zh^q22@=5sauo^#B>J8tPTJY$XY2;GzMX2z7M`9hQk$P#l*jK@IFpIJt*JT5uTy?4of=sxQ1rUrbUCf5kH*sw{b)4qv(A1-bY1 z9UjfpqKk`P3e}EXA~v^ALseipHr^XU^<(vUbeAW;CH)G-D)-~3w2^G2%~tG~9!?z6 zkK?GJ^X|ArCW_5h6C`cN{#URo36h{srm3MA-B2Zdh-r2*bBkvC5Gb*(9+x^7CCM{!8l@ zT0Ut--RaK40Sgm|J}AQ`A1lGzZrL!+ct6&}?uKVM9pvIwZ+Le=p3Vxs4cdOox!9I! z!HATraATf5dp$D=_cX4kNr^Yp5%FP2{ zAMqPw$`m1R_gu)hh$NSM2ce%eOwU&btDpz$`-@!6EHWf_7gmD9RYM-!4)}C%9LTB_ zLhDraBNmL4yjcV-5uPft}gJIyBe`ttjia_r_MW8g3!Jyni8u8}@ zd-YGA*SPD_AnQ?JbfpK??GIA-V=L<03S9XNEfd__@ddx?Xz+zIw)3v+6}aJzDvB-e z!~XF`bTB4|Rwh2SGyf_ISa^~;I%GjlOeRyk5kb!sDe=`xsKp-p@vSbsN~=G3LU5!obNa(%q*-jNIes~V%sy=0(y!H4s zad%!`F^@hpFNFB#XNWw#1PRvjnbDigv^jamt6T96XL#+Vz8lYwNsFhluQoX>@?R)Z zU4r1hDG^dEB4I?(DNrv<#0Rs~uvkl;=DeB6iX>yfW9|`Hpq@)Mz1N{>?-cMzh!QWq zT*H0_YGBlcAF%zD6;BVGOz+zS<4ju>EFTyJL8HFn$4enNC%KcYeeF{pr5Z-NWM{*D zDG&O_XE~%vkL2n;W2mm-OwiR|M@Fq}CVus+xh%iKwx4u{v%fO|v=1^xb2XkZX$@A% z+y%cnacbMS4#bDdzwahN(4E-HZXDT)GrD(Sz!npFw7v$ru5;L&@}22APsZH|8vHpM z+T&ZDp`7J`#kO0_@MMbcrQ=0VC>Vj|a<52bR|2Y)*5P%3tNP5D7J`l`yNA}+ETQgm zCv1Q00L=dbd0@5xY)pjYZ`32<>6?%7{OG?dD7z5i{!9nqiagx9e;8f<-!$AibuG#B zOu}Wwqady5aQz&AYZSUza`};&C^=~~y>rixBwdOg(&)3GsIm*E8@FRh{T1{K9nu8) z%|zPmfpAu^J<&)tf-zq1@M?Sxnf~S%@xRzhf_NzjNeB~On0yF={co`Z@|`^&-Zq;*I$3vMh{RFM->qClN!}P+V3SiT(M@ zaj*AtVZosxAAFQIw~~B-!H>U#Qp|DGaSekjZfo$#rO!}(`$c`{FlA0Jh+@K#5p-dxMSRZc;aO=%8V4@Lo#?spH`vSuGJVf?hA(HxU+Qy z2VnM}k!TjY%Ptl#veQqihWvqa_VZODcI~7fj2w+x_phnZO{+0L`zXmWP{rHh#rY<+C0y-$ z85%wpqcgO}LGJTmTxYI6eL80m(Hr+0GPWq-y4sg`=FxQ8IC=-JZPlSBTLy>j1$E-H z)P^U?=<{8(XE4oy7L0s-5MowVlQ0Wa{4?Gg5V%Dsyhf%f^; z@F*l0Yg-~P^?n`xmO2b6ALYO&^9`>3qKr*io6$4+6KQNTz;zS8klK>5Lw9iuhV&)l zuh0X+JF?pRWA$|mdR+tW2Hvuh({;suGvFiI;EUg!C~Wl9_``#|;^8cGdL`dWl3s zF&GtcZjzeGl6d;_N8;U_R4Z#Juo=H-2fT&^Dj7%#LT4Gv2Th>nmWt z%38SLejcnn_p_@{>fmyzIV7e!qvQUW?A=FWo)r6lnZ=h2mKud)__uiWy!If*rP+eI z`7Nk%>bCn}>jGgdhw$jTe3Y+f$6E*Yz!fQf*ir{1@ZVoD`e+6?6{M@U5akW4+R}kB0TTF9sG2~946EZv)lEDV#*8`qD)>34<2^K)|&!|v$TSyeL`$s zTLsS?7&duZ;+1>yB;Pp;L>4TC(s+N2zqJ}pjMC+ERYL`J@&?>1#+%C2rVG_AwqWxN zW4h^dKkS$}h@@p`I?DjY?3u?k?O~eGs~I%*Jw<4N#N2w>Rh7vJUp3}3uzb3 z!R?Y!rnA<9R8+-1g?sP z{>evZ_N_j0QhgS>NJU|~jxpWkKaQ8^j^@iX#`0z1ix4)(u?)vttQNZsaTYb$n5)3U zL-lw?pdK7)^@FDL;e21kBx>4ZkDt`~(0s2bHya^A+{j@JksSu>TQm7Y^Az~J!42s7 z^%(S2n{&}#nD_QLY>zz6jz;^_e(9OCrey(DTN;bq5|VVM(GUkV*?B16H^!!oBXOgM zF+cS;pQKN?O6>E4p=aNEqT2ivH~s#N0lHqe@OJ|Jsu71{YffRap%8pVxZ&=54Y>79 zDj4qE%v_JS;;qjmWMZW**y&|plEhRn2`PnL$^#%h(FE^iHsi`a%TcT88se=;blrOZ zyes7(Z(1nIxCDb#lL!?#eHF%fWn${gNX#%g1Z^GRkmD{#^e5aA>=E^b54WOVYiI@W z^ez`PU9Dwrqpp(`!$#tTjd9q$Mw~W$m;h37r}2&H1wmd*JB-*V4QlsPAs*t`xb@0( zX__s00=`QNivTd@w3P@9=G~qID$woI8PS8Pbt2 z^%vv){FxYkJXR3$EfYunkz~zRl)-j>D0(;sV!qgNoIQCyN;%~b{{lU_N!1Ka+-(q^ zZurIy`eu%&+qTiSf-K$u7@f$kxL{!Zf(0>4hRc zrFp@EPIg6aI5j?Zl01654{fH*COto0h)Ku>xIb?$dHQWgOIuEYz4qB;_Q@&iO6Fzw z6J3BBwSCMdQ3*6=sz6;rCs9}O0E1`p+_7~Ai5(meBrk0s-Dw0 z&4gL|&ZDyV5uEFP3jS3nk+m;9@T{2-V$L07kE4!|%e@IC-=;}8`TaG#y5S6YR~8Bd zV+`o8_{T8qmMuhxrIK+|0$AI&*+l6R;_C-mxNqcgXuO*ay$|D=!fQ*6D3645o1T#H zw;P#lRSEM5w*c2oAE2q&jWkMMVF!z4`R8tAuJS3MUsl6%7vI1m$#c1h+)8#UuY>&Q zI!CVFR6+lmX>?@j4N$Vu5Ei(3vzK1hp#MA>7QZio(w$0Fxph9jVX+NYh$Uiat0ABK za|vr1ejUx8%b{`U4dESwJEUlb8iwt0zzGM}!|?=po^7v+K5au>^SCZ%G_jmrd@9Aq zNB*x zP7gq^mKyIHb!3_o?8J(ddj&SZL#*$$U&1hYlKsjL9P%ztxk?9qgpD50SX*;(W%N15n$Ujy=YnY}kdHu)Tt_@lEI1rF30-XsR!2 zebJ}j-%1P*7*p6Q0Xdew#7=f9Hq4F#r{ZfMA=v`Xk1TM|!i_x*_JHcCFNlkr8M{e+ zA?4Q>GD-b5h7^V~t(dc5`^O$-rSve!G#p;+_d${BD&ezp4n#)L1s12cr5_SZe6Y>=Ze9H?$^7$FUJ!-rQ@98p3#ShYc+bW|DBAj-Z66Vce`{~xMoBYF_tC&3 zj+XE^#0Lc4N1^xnM)pF(3KBIc*!H&X?0G^4?7aMFClKU%2AU2RZ0a8kd(Pd0@rOp^9U~d8>syIG7tN&l zL;OkhSQ#QTmEs*+hWK%_OkB0xltSh;Sg6q~Sa(F04&*nH6PIrb4^Pb}9&__S+}Z}x zR07aNw-NI_rEzrfO5)?)!2B;-^4+&eq4IMEhI|-_&9xUWKxz?p5)-9@JJZRS%=0Ap z_y=5UvJMZFK4FesAK{eAGbr#L!yD#|gvfGJJR-PE{FBEM-POkhpT|yv9WK{k<&r|| zD;*$JOpne!7Y{j4ve?yaa`5r{YHV}5gTEzxp!4i$oPQgKI1@d=`x$=Vd!-avpdCsE zOy}}12HDz%LS&;9@QcMEI6JjSpws0nNPagKbd7>}xb##mqB4kD&ZgAJxdp%698Qam zOho0O%xJsEbJFRRMuY#hVem##`tPhAWuYhNer;u;+eXzPCVV?xnv&1+>-=d`*-31D zeh81mP2>mb@_5!%AG|Q^BVOC)hhu(xL8~n}(050`bu^cfXMa}l(Jzdsl$jRwxVfFr zd3b?ag&d%#Zzl5brpYuu=c=$u=_swOekE+QOrZ`>GU(quTj=ay6+X>%4i8x8&0V6D zX}9bsKH}Oa`u2P&YAuPO*Tc%`y=7Va`^aLN^)7~=iTlNhD@>rf(1EHC&0qDmB51PZ zRDMP#hDzKX_J7}hDbLm7Pb^b-*kCIjs9%9cr$w@uVHqg%cLiVXp-8?HSzf0%5@TV3 zFnohD_t?7_qNcou2@j`YlS3>w{#ee$T8GnB_l|6`o9>TJMqnJ$nQH?h3B&p3seUpE~iAb3ClO_Y~(eeI`Y}pTZTb;~+J} z{*}9_P^Ga8_?kIAu<+PUm|@?{WLyXr@81e*{hEkpN(H`O-2_SQBVgdOHne=vpi2K_ zY4rS={OFwvFqn88kAAF}cG}5+Uujthmo_KBfif$$S&)HU&&E@)XHTGce;2A<-HVZ~ z8F*n~1YDXko61KwL)Q1SjZuG%J#BA!Kze6? zV*UrsNVVoj9%+z={ZSQoLTVWVHlcIddf41%M?dEs zzz)+{5VB+vb>H~}7X22Xqf39oFDk;nutuVg{sRrK8=(Hg@qF2gT@ZA=5<14OXTp~U z;p~TGFmii<)9NO0Lx^@HRWN@{4U`>tNqBAXk%P(m)-98KG@c{2wE|djW*v4D(-IF zbjo6WOL+u`;d9}d+DUTDBpj;kBQdovo~aG>JfKRN4|WcNU#(NPbBiyopTC{Ga6AaN z!l!T*!z8S6%7B4-54d?Ch+F+`xsYA1Kvm5@LSdO3t^00_ZYCyR-}VG*BxHFE&235XAFxOt5uDa7fDxE@rJ~!qw;?f}LLdlTk zG=Yc*pBnZ~LwhVgw%io=?v>&928yBoN6gS{ z@5J4rCknqktb*@b9qIUS4e&{uz};JRyz{_e+%$0vdmHS?3$NsZS+9`HNu5o-5{K9v z1qH6KHWF`-eTIXhH}jJCYK zKRpMAZT^DQA0~1$vAZy{a68%_dn<< zDPko&RB0zW8jInQ_%gVt7)P889>6w<&qMvlqPmf;vsjc?9BX$QPVyq!Fm{MPnxJV3 zHU~$5XL~;E^f^LL<~}2{*ObGam$h_ci89rh2RINwLEU~Fem9Mz6VEtsapOG9_Im>V zb`PBo8utW`w0BbD7zc8Cy&SC?EXTYJag<+u29{mN&~p1U8uIHm- zChi0?kjn)Pg6r2tSf_lG*<~dP;>r|2`?)i_ z6n#%vSZ_{$ZhwMV6YrzNv&ZDui?6ulr7~~6;fmACG`Z^AB=Ydo>7n`hFxgh%3x7Nw zkPb;@ezai~D%Z_LH$7=@eDl0#i8 z<9G`$Z)AZAn(ExNLXo>W7*fLpgr9ga5Bs+l;O1%e{I1mtl(~5ox2e?(v5>v^*HVP0 zY@S3H#@M3yL-c*IdpH<>z2%|O$i>Ot?P}gryH?Djm$)=51#?3pNY3T1&5UuD-?_c!B z{_~CKeCz!XSDgrR!beeO-B~#0a|4V?rSNh=8Au!_@NdZ!P*GOozu%{mqDzr{)O6rk z^22$_V|}pd)#jJIb#UA)SIoXqi~|3QFlI_B_gBln=2xkF#jc_JXNNcY@5MuMwlJE# z6`urnS_)==yo8GqE|7~oGI;TG0GYBV89Iw6^Nvfwylb=>-KpXS(GUJ3$Cc%I=&pD8 z%6C5WMCG7K##Ipi;)BcUoViJ;BGrk#OgxPe*-(rIb*EJ5miGs+Zj3RFuwOyG$J}eiT*9uIh!3~HLmHyo zYN;)MJv&9X+9-%?zMhLKO+$ruPk)2pNiIC>K?lmX0beCo$BQE4VEzyfDd`%{2QwA< zG6TpglqO>L&TFY{O-{xtX%vGv(+OoZK@l_N9gfb&%1c@ z$x?8=I}=~;Hm4i6b>Y4zaS*d*C$>&J2dj4k{I7xY4CM0t!nh)el&ngTO-tcl!uLM6 zmPBY-B~6t|MJ1Ick(rQDLW-i45ryx4?(~z2hK7}<(J<0bO632%e_lN=&hwn>d_EUV ztv9&iG!|Y5a$1JW}V&KFZO=U^!kRlZ!8o z%5pCiBYthN9o~E)q8oQou5ibQDcua?zQqgplE=0f9kGwURV~7Ih6O}z)F}RJOrfRz zRXsM`#GA|rSciUsCpu@V6K-Al0uIep$4Mrm-i@Ne8Svll5L&PWMa!B;(RC-y!BgKGuX#Abt<2zChZL`{Z;@As zp7w5#v8=>Lx%bhgKN1`TFJ|o8^MarBlHIb1!c|H^pgr*`JW@)*)%FKy{@nx=*F6+p zO)A6RH|ArwuOs;`WJ!P9B=iJZIz zSD|Sp%{VdyW_9;rRB$Tnh`I~?^~LD8JPCKFYVw}zGkLm>52S?Uv#NKC@rdbrvZ6T< zd$pdj`9`w*n>Y;8AD2UNwtcaYU#zw^g!%oywcJ z+?Hfi74o=-UCTiyjusE>miRyQHA1DhRZO!bQMPai2)NYY0*bh#M(3)zT|}hUu?ex zUwHOnN7r5$uN=S?UIjwXW^>-sYC}tYU&1XzJW#w$kI3uHhwEuW_>#uAFmh-H)?fbx z9i5txAl?T-!gtEse=8(j6K*V1oS91a1lVppgjlYU;Khps{vhHYrds%+_ZLYn`DF!F zigAH%tuU-~P{*=lbMD~KBR-mXp17ZSi61U^wy-fc@s$A|f3#a<5T*;o|03|d<8Z#z{vJ+saigtwROvQag~_=ALI;OF z?enRIkBwtc?apLsqpipr1}4zLt~11HcNJ_HbpTIZwx{>jC1Pu{2KzTw0?K>~aD?SS z7Pch_y?+VraKKM-z|uLSGS8oA>#XNSb3J(I{>|)#!6{h%X*La-REKxWmWr3`P~+S8 z-i6@@Pk_wF9`Y@04m$N`L!{3d-Ejyigwk@0G{?q|T=e#vwF_Rn}>Z7^I?Gr$7w zMHYUVfaM`^Q1Pn(@2?SVf1-MbdiE7lM>5FpxQA}BCwQ=S9NTn{p-lC4i2uEUoih!C zb3!g_J9P^GoH-hwI6t#+QMfGE*~V5&TGC*g4VtYjCWMz zqjxRhk_&FZ`VcRkzUwpoigw`kuG6^Rjx+doywD}kI+v?83ta&2Hr)7%-)c?1a#<^xSp2zsCRZMN1Fr?|hPm~R z$$no?tcXn~jjj7(Tg!gzyl%k*E+~TAzBqD0WPm}W7`xLSlotN+9JOY(Hd(@3z>uOX7}%Ro+I0bRdG8hq8Z(ML_? zY+CjuSRQyn*qJ>^d4e_%*e0-~8aw#J8Uy+uek640cZxTf`htpTB4`?i!oE{l)O*9m zQoW`PFiS=Q13R5TCMpC;+cY{l>?N+QwdE?4v~Zf`HFhw?$#% zo_DMvB~EZf_HE{yUDk+VY#w4t-8ncEHkCgy{71$#Rlsq9Z5kNn0+prTS!RwRJ#h3f zypef>3K`Ahb5em=ce4gv>+=bwHH1Ut_FgEBa)3j(0>~;CWtg0-PkWDSm7)gu`4n9q9H~qF-J`pzY#e zTytmunDr^rlhYOG<=6pC{j~y>l|NwC{s@|$9R-W7u0ff$WH8T6M;{;OlKnpx@!^Zd z(CnyEa$|iTsGdnB#$lf}6jby9O^y2-b4@}8n zFuUv-jSw1xC7lV->S%^jPaY$4wi<%=hPk|fWuouuNL-vy2L28UuywZwL{6H{5{*;H zk!|n5bol*7 zFie_;$q6gDWkWg!tUt%r)E*aQsV75!+&p-jWJzg#Yg;W|daMVc-x|>P+dU-oem1yD=`p*Vmc(3JiANrEq>{rU zpk3=Vw2zyBg^~+twCN1$oK^v~tJL|6d0=6g&Wn*dvahLf2um8ovIF=$*ZL9t0YuGGt=9|mrtqUv|7 z*HneLBuySU=ON7Adj@J2^gvqtfTio)BQ(q1lKS+>q08E22u<4n5sww}M><*uJd&Gp!rVR9@(nX&fey8L3HJ^gyF z6ThoOKtOseUJSS-{?i}9m2SSqy|QZbPD&83`|luqqN$BinkIaA$uTmzT!UZu)<}H& z+9A=&mj4@d7gF8}-(|ll_}#FQxon!jc$6Z);ATU&N(p!1K`THrrh|10UiaDf9dydM zd&KN!8Vi+d$5j7?%y-u((cfmlbBaET5Gls8#OEZqqJh+#wm?u;70Hm&zym@y8g5qw zhYtUWh1~bP8%9Y2Ib|!Bvh)FtSCH%)=Ime`e8m+WN@zBo6e=1mc;85x%u%h|7I&7IOq36JlUU(9XC5L^n zo*D32BXqzbPnE|$wc-404i2X_aN$J=x>$|i>D`wxv(AZkWEgDzo}>r(GDrq=tj19&Tuj9|si@kTLU;d(r5WD_(;22?kr+yF=l>3% z%jrLG`@R$pt4?L3n{+9?Ka_svYP3}EJ8tuDMac794NsHjK`ZV3k?!d}+fpe(us^ShFXBxEJ|@lAk1YufBs@ z*Dj=@b|bElQ4F3}G?-ESdwg_d95`u~!~Em7@RU?D3%y{-b=B{IN81*hHM9{QsXNow z^BJTeQm`?K+oo5DP*5g}gRes@(A)c805*Kwz(HQyPOz!4VH2SPS^G@1Ak54nI zhs=aquXXs2!Pa!j6CpRq%E7!GA9_l*m`r^13l3XN!k9LMKR?!ia`O`UCHg5jW19-{ z_d8*{yCIHkxrsGD`=HqSB|K;zM-x;TENn8z^}26BP3v5x zGeO<^3OufIKZaR6!=VmQu=De5KH_K{uBdjySKaaOV%9KvH*5%3&>n!p{VkYj{f2c( z6_8hN|61snJ_fVM2uu-to}RPch|7{CJi;mnO*-_!$$bvpzVQ?WzMa7BawB;~_;k!R zbYlB9o8ZdVQRv+GoK3K+M8)D6B*8@&8eO|s=-EhIq#MF^kE;Z$W4~cd>S$admy3p% z=J9%c;h%lPDH1ervUK<`TQ>d75Nc8^C9pVIkp86E;f}DU!IhOS5H;2V(9@?DRtmiT=PL`nl%3+aU1j6 z$HKplb+~x?4kFhkhju1T#IydQFl(BKk6u28jk?92=_rbCTt6Y~e&_ITW-nYaljql> zn^??k7g3q=Xdd_~6q?dw;L3-yEG1L~e)&as@X2r1_h~Ra&EByc?LSDx#~Snx(}%3} zu53_^1`Qo7%!c$p?7#4z*k)h^MkYl-S9=bOurI)4_fO!ecji=yXw$osGHA-6=XmS+ zR$Ox_RwVwnpmd}C0Z>|c8U7h6!sf;>2(NhxT~evy3DOa;vimD~e#wF5!p>0kyoDKT z8H;Jxr_v^mZ)Bzb&)NL-6Th)tO3zP@1i7omu=|e$zwz)U2H1s??SB%e^5%`W(%2Ln zuhl?EnvmCw{tW0Z@N~C&ne_uV{B&5B@4YgM`x=#t^)~gw`KTTybt#thIWMKVTL;6> zug{p1aF5o$EA)+|^uzHmFPu?y62JB|v%sruFkR?aGt>=&DRTuT!>keSPLe^(B14}0 z>lJBn>4qpe5tine!^|gAR9>_Zb9ZU-M%60l(j7*%2UoL{zmZtcdYQ;gdWuSk zNN-@)>r%i!Se<)5^cR1qyoKUCOY)^~B7fmJf{*M!OhS^fNt%W=FA2L0%UUmkZQTTr zeW)u|cv>ztk^jN!hbw?btOW1w6!P9(O;CR$jwg-tAV2k!V1tSZ=$YzcchWqJ=uzbk z%LSftNg2s@-G+J#vUqR7Vjk>z5v|@F!|;ea;`DYo$RAY0O&#m-d6v+z7t&*yBYm7} z$t&&m=TeL!bCfyr3 z1de2mnUmT)V{8ceLoe6{leDgPXAC#%S?} z&d20?l?p#T^aNUXjDqxj5$4^nrR}1Te3!uLCx4m(s_m4jHqYbFUXLW+EiG)M|3_l( zy&IYAa+)$ClPsGSCd$_rdJ#QV)2@K$fV+(7q@RmuvqTKGjM+}*M%{s>nPHgqqXzZ2 z<$=PN}n4*gQ|(#+GhYKzL<);oYwN}UN2T! z7|w#1-@$-@HF&i08(VBS4;sr{Fjrw4|F}Vh?~IzurrBxoGd(*>j?Ny-%681d^+^Xy zwzVPaFFHjw6nU|+MmG39r33vO1NgQgWqzoC8edg?05_Nk&dNx2dhDbk{TGqUVUK%mp_K^Lhv+E-;> zHKhoOl9HgrBNA+Dzv9kPbH0D?PvPx8AWE`*kJ6@@4opXWjn37N@? zWJi&ek#GON^UHI0@Avy&XX|>I9S*XikIuq4^=BdY$*o3{>^`RcU?(WooPjt~E8Gyb zgR;sDam@}RGDtbWo*5)Tx>OY&pLYa1Eke+~G#PWW@39F>`dC$G4J1smp};UtyzQ^e zO}!FAN~uTq6<;?)pz$A;+w$ZM z(`c%KOJBW7Lq`o;r#n;Qume2)8pZw&egTiR2yR8zJa&ho5`hYCoVlH2QPl+^J{YK$!nb@wrniG4Z@;SzH2J)tYrbSDUL5Hu4Gfh zwor%8WjZgA!DUH~MhV+9O#9(xEVpVRF#|$nGJU3bLKCv9Ny@pPumSMr~)S+Ki* zu995tR!qLFh3+0FY2wmR;Ir>MmF0S3`|RS0}55 zX(+lV0nVQB=WMqvrdck}Sd!HedM~*ZQ^&mJd85OS5gx~0R|nC#%km`hU>2Y1xRQK_ zmtuQ_1#TW{q6&8z*eIg|$M!~Yb`Q$A$Y?p*80hb>ZT`9qW4Phi~GR#%&%lh<8&!-Tt{R7HFwBg zn}~n!jUnMqFFbrG2Ko|KW4~ev*f}J^YwrlWb`*sa`Rt7ICRn?om4YXP&@ zqj%P9PO!{}4aaD*_=wd^qdt=a)84>Ehqe5MX#-$kDv64sk<4JR7TMWv<8S{utiLP$ zE)1U6;vcuGL*e^YHX}Wp^UTYDz|=}!-C7N`Z#uIc?FGn!9IiwbDzC0`nQzYayOFQc=sgTPmI4=LVL0G!bZPVXJ4%_W5W zOYLW47L6ivrz)0q&mNxc2_Wl;3ak&?gbOyNv+38A(5*U=ms2*Q#P6+eNu zd(plFo7vZkds(z_E>|ILj*^BK7!wyE?IU|=syJhkcdxShzBcIFHJ@gV(q+j!he8*1 z$T~FvKMN{iM(r4o;tg?<_BZxaLKj7~vT>PP7QT=hW|xL9@`LZ5G082dXkV?@ct7MD zTdF+=L?WIt8U1`*dZ~xCtrLQ8wr(UNI>=qI+JX1f1nIY&3H^1Sg1$%QVqM5#maMi0 zQ}Z=}c5TDo+bgl*VLE=eaDv?L2k=$4yGWsHH%&@0$Dp+t6sUHA*;!A+o;EKuo>;`| z%zOf&`4(7fUdndY`rz&pfqc)dC|F)M3gcJGlCaM?GVCaWmQsDw9 zB`w5c^2;gzx;)(#%*In~-yy*IGg~q99E&!JWOqvc@>h~;sd85Vg}p!7*mdG6H~Die?QhRz^P9`)q+%))8UKg| z<&;?P-dpVKehvDiS&!R}Hp1va5wfrJBDcxk^xA%}BOJGr9=xsMw`}Qv(hb?5pzX$` zoSRJFy_)EzXf`$KsZjIU7|O9creC?o5tgkC0Q>z>l#!kc)7RW3%_)z7yB^9kR*InU z8&RxTmW#H33-DsA*SicSs%w)@_cu6L(Liy( z^_b1t1(2sV1fqM)K-2C3d!N1rWMhxf(%Bq}oA@(fwixm`KQu58q%-wfa7W29im=Qf zfwwCeM*5M3_XB!Wr$HgB!@*eJ1-HH|W)ElA;$$`tdy>z<-@IjPldC$4g`ME`A9SPp zTV&~CSSJ%{)Ig6XTj^NY12A@(0r_nvj*TRSYO@&Rj zQZTaRB9m9v#DFw;DCiMK)tqSh(l8rabu%GYdsAa^eg;Yiijjr6G7dbS$t)yd!Affh zsr8%UWx+U_oDsl3pQB9%Xu@7wdO*lqMRsPCG}gyd>$l?{aJRY9_=OswE|q6b#PldF zVKc~2)&Qw56Ikp0L|8UC9=!HFhTsoMqzt9_cVrIyDK2ESAuCaAYbLgw5az;KS7TCo zFgUB^BB={ODiKjk`a<8FA(K%p!v(B#wqIl~0Y*Lm$9_?L)&4OK9BIWBAlA2?QE9adp49 zp@jA~_#_dGN6V+t`jUfu+0~a!NoXv5(~zZ&+LI~wYdlR|T}F?@AAXg zg8xM9uy@B1^M`cA9kexE?xo?t@NV=gzgjcD*CrE`=f`ITu1>yYrgHny_N z1Lq9Rp-KCl!!m9)M&YSDSl~-CeITS zNXe@L&AXFOJK-Ts=moO(tml%JKStZRXQ;JLj2)K{OlS14_2`vOLB7trof7^V9bFjt2LO6VMmhliRmILv_aG+RpH;W3oH`3SqK z6G_I#m&h;U6jtO#qn@B7&fRnbm5u9Z!?6~c{%sE0-{GiNcsXV#*|BloWk9tpmsUAt zKv#P{U0SP)+IdrHIm||teeyIf=LlPPbpw5DUk$VFm9js(F4IgKE#}pDl{wq5q}8b= zba=#5&R?HPreT%nbo&-BXdQzJ4KtYNaS>MCVnLqf&3xEaWgO}Wp$AVsaP_(oG}%yz z4qoVl%Ll5-_@xrH&57f-=!H-~kt;vDpqc$nU4{x<7t^QU0+y2BOkwROh{inO^y@dH zi0@5)bDtPW3}nEUstQaD2?h7{GF065xqh~lCc@nawq<1*d|DBMN_-SeEau3wZa?fh zW1&CQ`#dN-T!~RuhgjQdT?)&y=X4ZGSX}IQd{exEWG62~Wl?iXJD`kypSEJja5373 zYk}bIHW*r&M6dmK60dGX-fhh!emxXSO`Z8wCU?OnMIPO(O-Mg)J}YQdVv)1$*u~q^ zDZWkx@6DW@td_J8AO&GZ$Bf6|)DCGstdB4wuuk8rS`K z+?duZ$9Yao;;rrCVc%9Y67A2Ssh3}{zVoxNVy`q#IQxw!6;s;!aRmj=air4=iOjCb zQ>ocmrqq~;Ys!RZ|6W~v%SsXAoJYfK(~-I{w}n@GXW-;0#tz+xLiKfVFwQyx&lc>1 zrbqd3Y@;6yd(Xg7ix$@RQxY`IHnL1VL%Q<2fgGCTC|a(9HU{ZY$nP+CbZ;D~{OM(t zDJm#5V?MJFt%gMhMXB-P?A7KV_ysEC;^UBHSg zGeDJ-VGDITcwAeEGpfxfc$692Y<>+c z77K!HvDT<1c?9Da@$ zaZ<*>du{k6R}BBQmyvF?3V8k}W)MGj6Npv(qLt}m(8SHO_j&=V8QQ~aCw{x<=*U}-#1!V ztR6>~b9&h1%_$_hV;TFpu>(cle`MEghJyU0I7(1ACpG67>`?m2d`v#UD9=&~I4;hl zybnOgxPHFS%!c}6r_rgm$3eN?jl4Eqg1F-EY<0u~*e0q=MZeDw-)cqv4LS7j=V|8j z*@{k;T2jaE8=z^tn_d626|QdgVZQOwa7t?jX`D*t+k`}D%U}VyzHNb~o!)rEKopz2 zmeKh12r3!=%W7OQX!vv@Ef35g^^XTBVgC@b9nztTRhCqh+|Tx(G)8HktDNJjDUFpj z=a~HLA23;=4K`G+z|BSZ@cGkrSY2xhK9jmxugWkJ$h5_*bB*A7C>+eyR?%t2G-?|N zXUdgoteKoZDJTi`%FffccPClgG z%epnm(PJ@8fw^cHeubG1NZ{bTWHLB0mK|LCf)B|Rf`U@w6mE>+XIpZx+E;`^(wt%3 z;s=oJphCe8H{kY}(QtYCB>uUcEbr_5y-{ns9Gmpk40VDx&{ywj+|w%#o^|=~$LS8Z zC?24hG4*)tM-%?LX-X*P zYp{0JYOJcxgKdvRpz#k+Vf$Q|fR-0i_qs>!DyOkyA%(@+8;8GpdSO_)SorNPcagcjep$L`!&kHwh- zY#{q7SPddcOJ8BRzo&uE&IQOX`iqiEXWfdS5WP=4zkc8nqr)AY6{W)l+c$d|`U< zzMo$fY=t%MWuQ2i203ls`zMBInxcX=eB`y(s;C))&=x<0xp)S zg^%f5a3F0xsLt<)NhK%X3Y&>>m*!*Dcq9CE`6&iZ@db&Wqsb4M}WoO*C1Z8ise%icwrKkxMknXmtPNh_QMz$>h#c zwq!{-T}a$Q1$K>a#qb8cDvn|U8l{~2b6M#F!AD32;IrDv@O4fxN(%enyJ>s5r(PXwYEd*=T#?4g z+jp3qf(OoQTS`tfzaYm}5;WO~7>Lk!;K8TL5L}+eB0rtp- zLZoa2YvGfan_e59>x-@Xv@ zehl4Q7*2!9nJj&U1m*QSXR6063V!FA zSNz%u<{;9QPbLaw)FM^^IhNH}qMAUOy|K`*c>}g7Swj1(P`oufAD7*^23w`gAbX+E zNSznpROf$CW_=4El%0gK8YhxA4x^J}v&bj*3NL?5fpSukS>SY4dKh{e>|YJBiC<@- zLG@aUaZA9O!ee;!F=}E#3>IClWjK-MknGLpAVK`(77l);&2b_UZ>-kqAD_H zmr-gs0Xm}^ahky-Tr=02H?>%WsJ9%MTQA7?+oFS{7d_J%g(rl4S=5b4CNO<0xfqt< z>}Cg43r*w~E}Rcv&P-1Z@yb2v;eVX!YdwVh`I4=(!9prG7^)XOOk;Yr?zxlI8&mdOf zKG-KO#dCDE&9P5XfHp|3AQ}?jmb@eDQ?-VOHQRLpnu7Wj^&P*lK1@}xS z2dUXcc*83Z*JK;uyPdjRpwk*Goq-s>H5FH7ZebPeH_%|+6WlK?fda6YrG1!!n>Izk zs)fny$Xa9i^V%QIpB{tmiw$AG_BtHYjljr-WoW54k#Whj;9L;|(Hat1_xb?xE#=&^ zZgD(Xv5=(SE`lGyuJj9bQrf(=^h@Ow%iSxFP35!EZAvJ6AG-*5Yu(2GIt{6bZAQzS z7u?0tZhUm)K1`E!L**Mr^tICuw^yfQVADgadw&HQNB8 z4GU9WW8I&5l$z+mX@{3mPj)fRohOKmbH=irLT%`@Aq^h*1i+@OI~cY03Uh2q;W#rN z*!dv~h5`hjFl!ofAE;xOP6|Q}7ms3tO1RwWBQuSBg|$KlS!uKZK2kJhc1fGKrb+{v zHfkH`?2g6IA2;}g;}ZDAORVw1xNRf8VJ>)QCxFyW0bHo6-Pqt�DD1TIT@pSH1KI}I?qiGWhYJU^X6c}^+rdr zOS?uiSy=!kH``#imk!Rox&wt)F6I=jckmBd6uB+4e=@7RCvd6s7ua%b32rmVVP;CF z&{#McJZBj}N}MO`yQhV_P7OkrUn0|7DUL^uT48oc6N=RqQkY*ZIPTLRji$4ZVQ+`8 zf>v<1EUj_!{1mh)umz32M05}-gx9MEKRGj{4{?v zq7bbwO~;6i3Or{KJp&$HpS z-t7hH`dh5e{ux$=1mKK)%dk(+10&yV(GQo|%L)ECjqL)i%=1x?Ugqtoq=XgN!Y#s+ zgB2inax=9Sjl=Z!ub{vu3kSAlH@r^n=hi|LM&I8_JIu21T$TiF8@ z%OToD8Kq_#vKV7wdRC-@{hurN!>+N|ziJoSJ6qC&Kr58n;EpBt{kZ$jZ?b;hFlY}_ zheLMaq+KP7-6OlPl(}PQ!q8-LS=ZYT@KPDGbcFHU{(N@w{!$dU>4pxY6HqTPf}zG# z{1uanr@zXgz%*wZjvw*UF45?~Oq1$6Ea*&yfxn5d0talq0J&hu>Mosz` z6GHWM`uNMs9K%P|@;iU#gVFIHZ0x`3%<=9b9NRIUCJM)!#2@CW`;hnX@+^}CeKU=XF@Ovcc8_SVGu`f%yd6=w6E1|x)E0!*h zrT%MCWG<6Wud^oNIn4^V5?F;ZFQnjq7oC{)9bIM_FqJM`>fpA^eT8tXeCk!7k88Z+ z*~jS;+~1|qTy2UgIi1=?Q3aVa`>GU4OUaR`S34{cOs6HEW$@lEQToD}Hrj=S;7=D( zgOL>u&dJMSt059p#9jEF{!Bb>@E!MR1mnxw@tD207=n|8IWcc9DBErYTR*JE-ebZB z>KPsl@2Uk12IDQs;dT~kx+SnXpCjS87Ak z2YXVO636~7)~0i6D!A~_6Zo!o5z^{_7J7uC{@5W1(3r{I-pfSIBN=$mZ5|rwEg_Z6 zotP$@MlGSE^oL!YVc*VGWQ*C5HenLftrf+H+0#JoLpeQNkj}hSG|+nr;WE2xcyy?T zC8kxQ{p~DX=XyEUIaMA{L@3f0aS0rl^9G|X$y2}JM|9R4jRu`hF*4%_#2=FYo8p(y zJbiQHp^M|8(rp~&RbE3Gvs7y9`-;y7rRnjpW4LdUOr!sCQ!4xZL*Fm>FDI|C3Wu6k zplr%v^t#31p4$z!y2$~Zj1sws7OO^OEg@3&<1qJ43!J{21_v%J;M)sy>D%>{xOsUN z)IW%2@*{b8a;6Oao$rkYR{M@{#zSn6$1-|(Gz3!Wy7;dXLfFmbK%84S4)a!DXuO@- z!TQWh_+7hhK4`7t8IhF34f?8r%vD>#53-3Mz@!hRx{W)C! zgUu5*P1^$}x0Eum4`b+uKrG(9aTc!3u0oxk)@0jah|WqHcqB%LWe*e~)`+2eK@p!C zJ{?2Hn&PE)DKIuJ#lgklcp%{niy1XI!tb*oqf3*VR4h@AJA!ta6LD3+0{m?1fy)fu-Qrv zo11Fb4)IvnzhWI;{#*~@a!tJU!aTOXISF@&Mw6uMBz%2K0xwOx%0!+kf$_ZCeEauV zm}H`hQx8dAZ%(lj=2-3vc+C;r1t3=7A!-i zwR;LKUNj%SiV2|m=~p16cbF51I>7vEbI95*minEN;9utgauDS(`QU0YoXqg;s$z_N z;lk>!&g0$rPt2$1$t=cMKv$>7C$VPIJ8g zh(Arnc)=Z%_V*t1t+r&xwfj)6BpMqGwz7<0mUzgFVd#Z#xb#^IuGIU@NeUHn3pY(? z>qj_L={Y9~I=hpZPW2>gUHA;!)|Y_KiZC1u z6gD{H*2d0m&R~YU(RgS>0ebc;FgvXsyxmeW>>QnqDi!ibM?3L+ks%ru2I1Zl{;*6b z5-0uc#HHG6FuU|0Zx*G9qh`*>S-o;FRdYSQJMkIaSB&h>M{3#RsD z1F!kG36f_Wpukrtq##)hLxK(by2}R0ch%!YiPwD96jPjJa}=(5++f4Saa3M=4__IG zp~xdmED9B)H)mc0m$(d7r>o)Z)25_5)W#Cy=F(zEO$zpn#upYH(B&S1EtjXzTUSTA z#@FNbynIZ&Zw_5baj?Yg4*%-G58mwC7p%Lo29paqn1gOA-fs`5qL1n1(sUgJ#9gSl z&yY@?_Qh-2@7S0AMflBVExD<@#nY{`P%vm5=I;we+YR05+<6^cMNXmSLjzPld6XBC zDZ~?|;p|+-1YC2y0AD)Y(?8oCMZx1n@fur%pdmU4lD^zw@l)K{ZbhJDRafBlyNR$} zGZ;thQ^iApfCERT;bG^CxcvJPJoF6MH`mwvUf)C%Oux!Cq(*V8uc+Y(9Yef$QJQv~ zIzZk*(`oECK|HLggp;gy(!-t`5cRAG4UKy7#i;q@-ua0?HtHI?AFhwh!cqoaOS`#4 zM~1mYmlAQy#$)L2`h<~|G2-D+e5&Yx$vZT#{_hlnFC#tkYN`fvIB(DHjLyZGp?Peb zOBhObq+zbiLUhb9!ZGXCV4J5aE;^}7L*3VKqE0Bvmt|wZ3wf$vYC!V~PQ&h_Gf=tC zkFVW7m+Hf>Gwc2PaN^fYys^3us!JDBe9>1pI{pd_x)-wmNlQpMAw~Pf6MRT1#>}4c zjU_QoEcw6#upV!Wd;djasn`lQ^ep`KF^FdZL$n zA{-tw1G>VEU~SJ;=2X8N9`AU_!YW)EUTWlGkl1J}z4C+OE{@#r%#+6VL$Xwtu$=-X zYG7pPL*C&-8k^y@obRzb%sCW@QB9N_Cbi$-q<*c!=klSnx-@!KsQa5$gMKX|{3>`GH8#8DOw&M!dK z==XJr!4Z&Q8Vw=^cj4Au8@j%HG&Wj1!=H`^@JiuLEM0mNmz?W`n`nDN$)VDsG>Pg)71(@9s1 zzWJ3qRxD<4*YdD`2 zLPrXZalHI3_D({Gb;usackgmoXw5;`5ER0W-wI?8PH2O|@oltrwksa`_7F67x1qDa zZx;ObA75H%PEA`3xgom+Byru87RE|ZoaSlR*y{mL3+5o^2^yFQ#_Oq8$)m#}H=145 zgK1EOo~y=S)D!G+`VfcH7V z?0w7mp{7mL+TO1}Z(29g9iIZWF>6p@RAyKIGFb#=w7U5^Bj;f=g&9=)M~9 zAu(leY&eqM6|LmwjFHE$+z|Gy>W8xrgt1}k33_u>9oEb=qZ=ElIp@@Rgq^7nSDuD? zQ;SeGNt?CG-@~U-c_47A6Zz?>%=7O3kxW0CIUPGhU#y3iipw+%Pz;CJW5lQ_X*JmH zUI({pr{TJ153yj$Mg59f640)d3u69a7?&D~jji!0>t4*SFY09%ZfcQ@!8YVelqm8= zFn;Y5#v1b*s23Iw9>)j3%y<~`+c)Bw7mwj&Ry{l}7~+4=jmFV!!KA7wN-|lgu&u;` z&og&rU&)6F3Pj+oAJ^fBP%<_Z+2S)r0~Bfe0u!7JKp|HM2ktk3N6RI+U@+3l@hfn~ zmln_X{p!eW0gww79_owY;R{3urkd zOc(!-Xdz2PujHMmk}89D&24dCUkuF6wr00pOVR!Av$)CM5p{080WV!W%CFVL0#6^3 z@fOw}meB&a_wSjTvM0&xP5_?sAP;_ooBvG3aH&{)FB5?RQjb94S}0w2XDH&a29nc* zF!aw_G7T!hrR~+sK7Jh;h*^%zc1cR-E7@j?XzI&ZfrBq2nCs75cxH7Aj5@y%`>zJz zB41rR_jHm$TES_2yO|&@cMnX|io}l-hVX#48KoA?FlcV)&@1s2DBx24R3$yM{62-c z7V~IRb`Vr`pP+a5KDP1JJ8b?po2)*61EKPxc)mFGnbl+rq#$g~mNK}sb|uE0 zSPSD^Wejvnq+sp&nFdNX#0=JC7O~YnSJ^D@kga;0UVnK_-NY>JT;QC;!Ja30H zm;OPUWl{7|cVte;ub`$VLs)aJil%lPpf~Ro>GkO?0PWL$pAoO@#N`-z+E zb$c5qtG~pY_09NmgrAmOm!Q{b?wD;d1AH1s8BkmnyiiSmtCxH5mB(mwQ+%vHs<4Lr zyKsy?3>ToAiwo)gS%p6&6G&rZN4C47mrJk;LF3OqK{BWsKPfK90f{|W`XGv)W;LSM z_JibmryR|0{YT;c-YBxhiG_$3k=)J#>J%B!{~&#-QFE0HJpCp|6F1q@`17)8Cb$H1 z`5^9vwIlj@XR-65y_}WyY4W-@9(tta;>Tw>Ecj`kPJwBxxITmiX`NTl*+=K8B0#ZA=khvF^tDI3+;*f|rI)~=(*_o_H;`Eyom|C`yJ zw4gaY#rpPsvZ(1(iQkr=K{@3VO7YPG?UJLcc5n(7mR@I>>t|3zaV&N6{t(%J0d{FC z^7mj6o)w1T&$eq^#N$!$R74gPg< z>W<8ygiyL1AVIr&jOcV#Bi>cdK;4PepkK$Za855+bSKi|#B=!WKnLu6kdJ;dw^7Nt zSzN-FN8k~YM+f+0&{!r7moGkJA55oVh4^CTP#a79A`uGu_5nM%hp5^$5ADYO1NJGH z0@+9QPkRZ=6?@HT=vvS-ld&XMo-}fk9NnOq@fa`rIJ&o#yREQuT?W6^J(LEQ)o_VY@3~L?aiHH4Kv^H3vb9&| zQg)LWJqtbp7cOq*s=^y!`P>`q)7*^|qqPG)@14ObMQXUDPaQw*Ujlh%qp+cV8;URT z;nu-IHsg;K+1Yr&F3T9Y5MaU%+kaq7|7BrFu?{KL?Z<7$@36D2Z^3h_Aq&}~z{*mT zQRQnAdli_AuLMVWz0;fB))!+VUxt6Q{&1h@2-@6M!?CxDnf`Src-MakN3RvaEhAa= zn}HYJ{;p2jZFAWA^km-m(n>mGR|^@R*Wh5R24AtD6b19n(mBV^xO;IN{kwmVZFnRH z9(paP>o0@4UOVts)mro|e9S~$%Q4XEGc$g>8RIG|*>;I$2w5G4;;DBbx$g{&uYJq^ z3r%N_l$O$~>RhPJP{#N*E!=yRJM5z0dUiLk2D(nwqoZOo&YnBOPUR|~gS9rV@v)Ot z23SE`l?}^RTZ}UM=HQnV51Ic;0W|Wn!u3g!Eb2dP%vJTkF*(n0->!@Q;BR`6AXBeW>q)^;CGX`aSRY>4Ehryn+Nn7#=`?Spw zCtKOFaV~eDynH*|ku}7lR$X|XoyB)vQM7LJbC}<1fgM)Q_-W-IQL#T8ntq3}1sB}e z&yG$EBSOZ=#h{P+|Rv7c*F{@moif#?ESo3`*LOmaP=U#=AFQn1c zN7A&>%9vkvTMA#8kL1iIH5F>diB1xHdwUbM zlnlX)^@2D~{vNc&KZ5dsPjG(hX&C)80ft?h;N_8x@Vjd=1~{FCRK*^+FkcP4ZChZy zrZcSRdINe3zQECwS`dFb4Yt3$1e@+XhqN1iK|#qD%pc0({8dfx&aac7`*t&!u8#%T zeFN+^=%9{!JzF;G9dBxPh1-^2#2@V+1dr5F?EU)%z+2pbrv3fgP{nKvU68=$ofpO} ztG!XodneO4`W)0pU*qG1#4#=^0SARO;rpgFtY$$TL_SZ30Etj0JAN8&w((&$*Up2d z@)#!3uY%FW`RuC7CVUgNvr)A>fDgU%99)dX!W6BytmW`9e|G6{xavK2gnu`%nz~fB z(oqN{E=l5?E0yeOw?FTCZIGKYu85cUm;-^+bNKCJ-@}R&CC=pJRk(V^7}A8!L*U+t fY|6+@dT?VRlEZBH8#SN3dGr{XV)yb=iT}a>uT`2d diff --git a/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin b/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin deleted file mode 100644 index 82a1e3c59749280ca3e32f4bc4ead483647b3fc5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13828 zcmd^``Cm_87sZ<>B^orNG$K-IP~EeyGFKESA%#SQjD@74c^;Go&4WrwgT{OICn-rH zB*GVk5M@ls(9{3%{Ca;p_q_I6>%G@;D@>*vc6&ozX<~NIpu>UP%I!R}#rXX>?4o zl~(%k5t}VXHL}|Jamrok7O3(?WE65__e}c}Fd_qDF93&Q7%5aEROf$)TdUu8@dFCNR z`u3y~-PKFc_(}-b-ERTQ`Ai@<+X=*q6^O5~6irf@3p(ZUaJX#=F3y{S?|Uzx=ks}V z!>R{FKP8Apzs`emPjkzEZi#!(7Q-bM(ESLf##7hpbL-+^m&@ zW#=QHX!$t|tNliFzHR0DehhLKWIe&n(;bi3NP)?YP#R$`i!z6J#C<^sY;0SFnc)%m zcw06+Dm8-Mb@_DF+b~iW6G@aL4Z&%lH*^oH!<<}Eh;L0Iv!?uI96uZ*2Q}+y-V-D8 zXYOm#a`7N!7*~_aRmIE~k#7t=+{|n{;ZE1)XmgV1meB(O`=Q_06El39xUwyUMCMNl zH4`&Nvs^jUSvwiR%6!efl{>(u;Yb>$t4RxUi?}4+(`4hldaC^>4{PM*X+VuB-J7n) z>^90`B(FSTgtrEgPhtF+9u!E|jF#DOuMY0Rk(6)}rj!az2%f$Q(TN6rYV zLyGZ2D)o#Ho2G}uuNE;HY8^|`M(fBI=TI`5kw`5z$yf1B!IQK9x$faG}C-7dbw$#h2B)?chbeFQ}=Qi z8Y=kBHJx#q*Gz`YrD1XHB|5e1BwUFO=|9fj zbq{EI=F!iUXVHGaa@g!KKrqRRNY@#{kZUvX>b%bJ9iC46#0;5dmSqrmt(~0FY@qI? zQp~L-v1Iwy%g7&=MUJJ9)1U8ZXzd$c8n0svyEaV4J2mmRtz-$DT{#zycFurpK4#?2 zL2*=7)B%m3s`y>;HBD>T3=TWR;i1_NvY|T+*4axjH};%?@V*TAH*$ekB~QUnvw5&$ z^AqN7=u4t{poPq=&ZLWNr73rLE!6e;G21_T<9jZF%((iKKP47m@6vE^wlD+@YNOER<&g52@d@-fLKXmJaeuMS2|y({Kph=IPj7g->S`Yb(%jo%ecef<}@%G_6PX` z*O(W(A5$ycakA}|J8pZJ%(QPyr-#lcpmx_vwCT=avb;x_%O}6ldsp63MZ0-mv65@p zdc_WlC?DciElhfq59x0MQA3EwywEuUMZeC1-{I-_yS|P)DHV>HLo8HB9-{HD1Momc zDz{PYHnV}fMjphi1?{njhPYZ~^z8~KwzoIpJNHy%etFN|CO#boXzytVan{zjzA=Lf7gmGs?_Uz1m!}xt zwa>}+oo%GG_5j`+bR~vKhe1ndAI6$ZfpjrPYI*e)Y1LYT-Y*03e&lbu{kk%4`cX=6 z$i|G=6ECr&d6p!mC}4HGWG-4fOtyB*AuzX$yg>h6Q+8P z!0PN4GI^T~DK|)i>yJ)iUFFXP>jh`wT2>OV_x5KbjMJblx|{49od;GWS1?QJ5-x5w zrbYQOD1R*mE=^mEYXuj=mYh#;JZ%L;P4lPO%z3Q(7ms}p(m-v763CY{H_RFt;*Qij zC(b`psr~q8I_vQ!Jao+-<3A?Bo!Pnd9Z}|>tQwDM!RgrI5=m`~E@A7Ni>UBxIo!=U zjB3NFu*qAKH!LXwu*4QbeRntXTej0N#b>Z0@BzM@y^6#Sq~n%*t`v*|$n7y9vPX6@ zZ=ppwlM=lVJZ18+#V#Ekm2~0$`dM)LK_>C5K~Nn8Smd@1Z{8))m8lG0{exk?eFFSV zYk(0lN){j92@xlyK{4hR*`K$N{wB$^?ES*BSCn^FAOIECafe5`Vry6`Q5!Rm|H9+8gOLQ=Rycp9yF ze+d?HQs5?Xm0VG2CpR0NxK#fF}4?yKx@Sqv5#GXiyp**D+_>U#Y_$L9yVy&FM<3erx}UN z5pF}91#USijJLh?At)jkE`8cfg1!5QwUQ3(m?1(#_Ggmk?xJSOdTE$FEdmr~X=AE& z6ZO^lM9-x%AYo$(%QsJ)XPpM{KFFby%7oSrEvMT&C*!`1x%B-GH(VuLT))#i0!q_O zP+4F#_=T1*UA^bI)GyZ1vPuXO6}RKLK7G1F!x%9<79 z8n?AX!}Yw`o7jh>Xh{%#*WipD)lt+S(Gr`MPxuFo7`)Lej6%!`==kamKSoaDfO{3A zJ{U;DUk5XVR%sw!l8P%Iu7>d`)m+oiJ~%$v2m*5ZNfP&i$WG4yx3)vTFDC~l9|wuACHA$0GgwaVFkgpa5)_I$uGtCq~UMeBMe!=+P+buvt0O!@r!UJLkxS^ntvq`Qe zhu2Rg$9-pWn|%1-@q_E!*E=g8i;kD$SlQq zD2xG+Ha8M{)e}L#gP-Qdr(l7p9JH(!23}_rEPWXco`Vb-w>(CtoV3R?um;LI?~y0b z+hNH;A@;na4o$xi40^Xx=uyv9DifDP%CkS2$-I>&$=g>@FL`YyK6WX5*cxYIAE|}w zZ@eVIcl}X!tdWuh1U_33*wdy&8&&=B)AVrea-1jr5j};GTW^q${s&QW?HbJTl_ER* z!?Cl^1I!P~V2_P4#``ZrEqyy^5lP0VdvRo>dH#TNq#6bo4&p!gR zbU(qGl0p)2)t}C@Gr__sCuvmmHcT>GZQ7ijjh5pJ*bP^v@wO*7(yn4*aMyYcf6qR^ zOqVh$FJ(e%8|s-kv)tJAaeqnj>OPvJO|Z;x309cQLfiU!T=IMx2tCyX-*PQ5aMnPk zya+G)ttZ~Qj^UZ$GbFSv5Qi1`;PuxC>K*+rBr{cn{4V!r_T%mAn({ZhzlO02kXxfq1HR}aP40bX7acGzBvKYSO3VT z!qu2D=tc6)2FSpCXS(HbB2BEmOpe^x&j|~RHI#@~5uN{l4EU`=4bjzbX2o(^@s+{c zdE1aIN`v{@ZS?ksJvhfSja)xfO=i8)LGSsE_w3F?&v(tTaG z2^-YW@ZD@XZm>B7?@jseq{sxD?;PAAq$Zd5|k4_5Y_Wm-+w!Te|Y zVThzu*RMEpbX>+mQZvknFP{K}^X(+g2KGF2QP0tJ#LXf-y zeqK43ocv-11~Dmg-H8xJe)U%1TVa5|9Uqd^Xi>P}k%7hIom9Fd9M`(+!w888tUDYI zToZCr%zWu310euq2QY0-r54souuz7Dwew<7&bx>BCf|p78%?N6#cAp#w;n61j)3#$ zFgR7L2j3oT;Z!e~Kys}$@eG@e)4%N?-3dL!CC~5cb-UaEYkEyd}3GCV- zg}c(9(aOF~c<=lQhH8iE<9&89v6U*MckOBL+vtex@ypGsdIVuaYB{UAPmuQ@qZVZr z#)0skUZ~ylA4(SAMd|P?P%YdK(!W@CX3G*(cxi&FKO}J}>j%=u`e>IDmRF9S9RtaSdJ5I!M!48F7b6XJkYzbfAo0aY(5@1OTAy!ZYkfN?t>_}dvlhaW z4+c=(n~O)M%YfwGcThk|(PQa!IM?)^1Xv`)>{T9Ur?QTA&oJOU>axeI*Xk&q--%-S z=TS0!7YHs3!q((MdTehnE@;SygCX7!lax$bRs~a|0~?|9%2m!(Iubg1bUFF#5A^Fc zQP9!9#cY3QicxEdvFZSjO&xt)Vg3|qym*jo;|qfK6}yS7=_=a&PyrHUHh`qbA?S{Y zflq}QG$3V+eB*Z~HffW&_d>hq^kYGU4OPHqv0CQq%b(=d5gpX^`o=gu3_Cx5LBuQ*`ph9p8N`wsJ~6|tt@;|4j|ewrjT^8gLqhcr5%s{alc1}Veaez zq(_fqp2HDn?1?5PtqzgXa-zWHDG=qT1a8GMQ;Z%Gq`aIhaL`f@9}O(vl^u}ft+}Pm z+wxt36}Ftu`+7YW2Qy}|5vP@TUq3D3Eq$oSUa=GAy?QLg`m4&aC*Kd^A}2X6%RLjF z^k=hWiZgipCrh}=$+OwY%{35Muf=Z7(d6|toB=b?VI9vcWbbIKVqLGzWL2G)@=9K; z0Obv9uqba+^Ah&TBOTV%MGiB4jo2MWp2E{s9bROzDbJ%*hMm`P2VNOT z(w|0+fYzeC+_or4R8`~M{x^*`IR4-NJ^!&xx(r?hIe6()GTxcUD>LS)Lkm4b6K0su z#FwvVp=1T!F{cKMIxR4B>1C|vbYQ0(%Z2uoka%-_Cb|AQiT1ZZl@tDucrk=?HBKbo zdI|`R@1cW>{ZaMmED#suXOHFuAn#NOB-#wpPiA=p+zirtqI5Y|+?{+ftU(7y47+ zT74GsyKX`9@-Vq7cml`%upm05iMM3a@$4YWyZWoxRbZ{3jMi70R*Xxe}%X7UO}V9IlO< z0~u#BVQNYMWe<6QuXrClD8tc>ho9h?*9o}y%s$e~J%OAn3h+5m9Kr)1z`=9w4OeZW z$xfimCtGC@G|Qy0ath=>?*@zbJ;YXFrkO#U4W87TiucY3<6qf8=+3g{#h2cIPYvC0 z@%-h}EMS%-S!4pAX!{NS8F?@VE|s9t>puNINt8 z6msZSt`;gTR-;wIfQ9~ z{djjwofYjJqcKhA@b8-#EVGm$4W^Rl`ZW*p3$D<>8x&pG9Wds06(tjj@Z`lRZjNmv z&fB*V*G4mF@mU{*Hg%b49PwZ$$^W1Rn)cYU(3xx6>_yLa^wQ?6P)_#j3|3CMjm8YC zvb#izQpJ?yh2PhCGRB+ z%)C1dq4}c*`=GFvW_JX^v&)JocwdgRPTvh)dQtGwKp$uLZNtYxdvL71k{)U{z``f1 zVa}ONxYbyiT@*AI1*Z#v#_)PvJpK@qL&s1<TZu ztSctipE&P}bO2`kW-y~i9^GdoLw4*bI;*XOz4<~Dd;&FKpW|^%`uq&NqgJ3epAqKk zMBy^&*|=v$51tD<2|h9<`2MLaJ+tsSUB0N9#*GT0zRo8qJ98lljrBAB!{@-ibP8xk zClX(|HK<2kz}H?ol=975S8EEiNCPHzYEw9JKmmjd7!^-LCGK(=7dKuRBwo^ z)-nUnr%QPG?_hJQKE4Vy#p{QS@z9hS&`ZmQfUcReRiPAXjS|3W!oSb1ea|H1X5j+OLcCIK zRe$xyccQUogu9}k4W6Gjs5eAm`P8O-W6%RK$?NdrD5tB1r>~xdqG{2Tk@x@|_naFtneTM&-B`Gg zIv-+ZWWXfnsjT|$416`i1!Vte@iOlF(n)SN8TqaNn9V!`8LiW>Mv;MvYlb}e8SxF3 zdsWffz7i(a^7F12-y(HRogy)d%&5Vb_oIR z;K@Yw-)2%H`x%|x!tux4w>0U$N$8TPiREV_aOA`_+GBo^eALgu*jq>F$xkYHVayF% z*JU?UdZtrmVpsS@CJfIBuY_5evgBQgARc~m4oxag(eIu|u+A=kWq)Sn5;7E$n

#qy&@Qw_0lDtNb6(-PHIGVn^UJ9Ct0_dQY5N85NqRC zVfE@7;&V_L2HvrBXX+ky_qPI^WHpoRDq_)1jfLlI8$GgF8blw9^CT}uf^|V5XiriA zBkd}@cjhd}JY0?cg+`jGS?*)xe|*KgNmJo;NFH4%BTeOtlHu*G`(%YhHS{k(0>+}w z5Ykyet9F}1@&Qr!vnUX~EVS_Da1mWLaesrHZc{1qs)=2d9@^|I#$x+!ES&C6&ZVx! z4@0ZTl#B*0FeRGwZB>HSUQ4q0=_6)-&J{R5zm#5n8w|VM#8ArW9b8_r3&b_IFi#6^ z5b33l$-Xdg)P21lmfcC9&Bd4S{lp!5*z5qiy_Um+W(2r=8Q4<}m>{Z01%ub3%$p!C zwMr9gCK!-v=TgX?$pi78CD0X@tr;JoiAF=N7I3xjPO}gI|xpcIGG4*dc&! zQ~W3wbq=HAUr*$Rqxii`g%xg`2e00>()Kx<5DKotpNg$$XHbdb`(ERYj1u~2*JNJu zlyFqLybZeOJ}M?5%6=Ri=9Vsh1(QzwzzwAxRJLv|YuXipV*kppaLy;XT`HWhi*%v4 zbeF-=BO0jZ;e)%QO`s;z0-Z%hY5VFK(5GMywyBS3+Y}LS=i`WbkN|Wa5P+IIdE^a? zFsgHGaA40HTE222ZjXs4QKKIq=a3GLi1jj^ZIaYsH~`NdGba@Rv#77&a(uVAmWdH| zghgwr;I5l3G|KtIg1-ol%j#+B*Cr4zn2*ESpBR^>9?~#+AJS7EKxddP_y{c~v9lu~ zBf1@iJ7@D&xv4;bBp+LuCq>jolYozZE`07dg@uj#vFo%1wgMl|CE+ys$@d{~6oM}P zs~{9_#JirogKXS13EuxVOn0d%fD`G3=JTms`{}7LG#1=&J?B0x)pj9Q<02tBMutuC zp3cfk`O=R=o;YR1jzl?iLT26!UUBXW5L51f`{A18Hn9K&?L(0NUn0DGD8tJOK1{kU zALogiM$$Ku=deXG9uCa7ND7h#+0Zdnu==M%dz>t(LwqOPO*oF8?^gl&o{boF5&r&8 zf_2TCAj3L=&Sp)`Y_^(ljq7@_M*KdrIO+?|8i|9ow1NE035Q7gLFS3-RXF)=6C)nh z&xm;Zrh|v}L2bAtJvV;^1TM2CyPiIP!KqK6?(Hx!$Wui1=6>cj&ZW; ze9#-Ag(DG{K>Xk)9NHm7qgLhNNuLl*)9S_gcOOt)au1YlT?+!0PtnKqF7EC=4X=iu zVM^pL4AvjO;^nP$)QX3?ua*;mttaqhh$L8!+$5uVEoL@F{p5CjG9Im1g0{bNSj&M} z)GD{f_uVf@Tz@i%Ikn;E+-c;$53zXAFbKu`a@Zq=Bjl2T47q+e6#tWFX}rmC5_@3DjvDYt?Z#^Y_n=jC95$v%vbx@48LGTdp^*{ zKm8E6&7F)^q{Gmq2N3nq9=`NPn|(f!gWq~8iSA8#C<-Wt*@p7S&NvIMW80y7+Y@-3 zPzpocCfu3#*U0=ox0t?t0N%`Wp6?Ax9kV6yWA-y5c<2nYt1pF(pQo{k50yc3%Ut&G z?L-)}3#9X8E!ec6-8jZC$WDzp0`@by$VJ~esJ<=;InguFc!MbZF}h4so;GpsBcH*H z`n#yJ@EnBs{ei}@ad`Cg8+iFUfw?HsXUq zP0lf=kf#1Hg2ZRxq+E7_XI1Y2FUNgkmj6+*Th$U88(Uzi?prc`t{MD093hV@hxS-G zkg%Tu1KyfMkB$>t=s`ux2o(F{jL!}iqp^@BzR)vlINCA6mn~%2vrY?BZ$0RL-RuvFc6P{^}KN!Br=JNY9GR1N)Ege6^p5v_bg;%&A@5bgbp6)fMCZAxbv-$ z`E{rO=FBcb=|i)yR%jz>idP^>{B?}aqbE$`sv^>FqchRRnAjoLtMH6yCafPx0{*VK zSbCIYemVL>PPREP%jfe9vm-#~QyUIu{%*Kjs=$k*J0a0ch3L!2vFT5vSsq2Y{!er1)%b}Z-f6{=* zHqckx0}59UG;DGmfxUMP*z*SMjO5#B*p{n@`;~>+!is$2uq=|iKNk&Z+val>f8Ls@ z2Ww#4Y5|CskV7S%LsUF?3>`k50+-wl81%RX(mP(k^&e#*5f#bY|E_^{Vb?ICa{wch zgJ}DQ5S&zF3m=s%z?NB0#tnK=DJ>i)F{8AqCX{S=yA;X72~V^r7*1RB@X28@qP)cx z8Szwnd0z?z)%ZxSS_xTqRE4hB_a$CANu0{*Sn|+a8BNVA&`kd~efL!yc?S2< zq`DI4uQJ9(x@kmlc`h^B=7nbk7Sa0$CUUpTIU=boO}zTfVa@#`=zX`1irPFSR)T!I zq|I;1tDtw*f9}xPjMWd*N*JcXILDzlowqJG^zV$Ki5yx;3B?4%NMZfd|iE zd~-Cgm!8nc!eNZAL@^w_rh;|-pWx^HD0u#QI+#c~&;XAoaQoqFu<1XG(S8Q_b6gr0 zozMWsS*EBPw+D75rIPiEX;Ahq+{{_80CF1=>FM76X8aT1v-s9hcl`)h9FoUb4yl6r z*IN@i;3Tf@yfUt`2*QYUZ|K>7FX)^FNnC|`CcbQ*2c{Ffi&{mhXSj&sKG}!g;uteqU?|7+>7&yL^KI8SY8>qQ7`md z04maK;g5?tw^_WnZ=_&R)Gu`jHh*FtT-C;es6GLg}n;S5Y6N6&{t!^uua=Mw_=`HrByCW___(U{Aj@+#}}I( zR_nl$3D>Fj=MI!TzYptrXL08*H4)wDqa;As8k8LMVb<+7On4Q5ap%{7pu~5YM;5`=`<%)-w;by70*1q73*VuSYERmw@No zXISvxHp~c30KEoJlAG#B8`HKzQM)Lstrf=A7e;fTMtN}2SA-hPc?QR{Rj5!{5Ki-y zWal5rh zYv`vpqxASmeV%8TB~R8ZAG2O);Gz!(Jckx7o^_uT`+PzR4;#$nS=pH5%TB@^Yyn#O zL>zuzFJ@MYt_5WuJ-F&E4xS5L*yh|yD)ZI_56*f*iMZ$R~d5KYiPRGO+=>xIB%1Q#@|C| zrmrN{e+Wdeq6i#f)9Jo<_i4cs2Wa{*3Zty!yNPHP`3IzX?(c`RC%?abtIDLu4*Uai)wKHoVT$5 zT_4%+eF=(pc|aPih3X-15WchlSIi$`uD#>Y$*&H;)zJU&qvAQZdHXul?#_fZo*Vfr z?Ff1=`^dQHYBaSiA;+ZWF^5zPiBs-53c4dR6sWF36 zOYwUCPpJRvjoY%V=$xH4Y;yfZjxSspcWiD0<(PY5J zsBf@8kjAym-bmR$hd}9tIkd}3!^~U!==P=CYx%k+{uSp!@R(?n-qgxA{|HV6+$tT4iBnV-fyNj>2Doow#fH zBwpRdb~;C=gxp)ln-_Vz9f^D*+XyGQyk}Y5y`$ps$h4g!D8<$ z-nP@PnN^R@(|CbL$Te>OyGK>T^1@+~zBCR>2H!$J#D9?Ist#Hmd9V=5sriLJw4*hU z+&K{r7xu}sdY6+()4@va%-t}U5qTD7zXjgCkq_j*qx!7&$6{i0I~G1Jw*tPjuVntb zVEp4eoxCh5!ZL?-WRJE8&*W(~6_4egcyFPG4h5z(*&&qsSZsiE-Solwr#!TD-=?D- zQ(3ZU9rVBF=i!@e*T2S!V&z@QH4jSAk#t$)RI6KdGva#ssZe z^w}1GEw8drZ)_*oF~QSSTtf-JfDgHMAQErg&L!dP;TYLoj?dcwKa5tBM$2;$QBy*G zU7GMcKSL-PAEb&tZ_V89{2?W>sVG!j40gW?ptC=V>0cqwX?cY}rJp<~*uBM~-74_w zgcnbu{uK#VdqgfzQXo$>G9c408*cQp)0jR{o*@4VVke=+dt|PH9e*z1l9eloR;&R% z;9`jS^Pba=!%t|9fHsP*Ekj-9Ul7#395&7qWi32Psqpqvth(z%e^>Yu_k)h8l@bDN zMiwyJ;Tp|d+yaG%U9iJs9tl;-#i+%fsa%gOs-Fien|z)+6jn8?zH<*J%~uBBIU6BR z`#X`UUyqklL(ywaGWPhb#1nlonY2ezpVsiSuXIbx*GQFo=kI9G~oV=4!X6f2s#tJ@#0QCrY=Gu8md_?+Q={~(vS_XqBZy$jEt9pk<2XhE}w(&%}7HIc2`0$EC8P%?I%$j$F1 z#{|X5$4h1OSY;3maYt(JK1NH0-;8zf~P#}v=-renGe7&+rFtyQ{5qelG5 z{>9@|tu`DCL?z(mEg8J`pEh&#_A;}SlqM=1R0KPkiz?tqArtXm{ubwA@ a{aSgf$gLQxtJ=c;i#Z1RK~6y3#NdD4a$&Oo From 44a2b2730489726707789d52cea26e848a4602e7 Mon Sep 17 00:00:00 2001 From: Thibault Neveu Date: Wed, 27 Jun 2018 09:45:57 +0100 Subject: [PATCH 7/9] Little Refactoring + Models --- .../webapp/{level2.html => ddpg-traffic.html} | 32 +- demo/webapp/ddpg.html | 66 ++ demo/webapp/index.html | 55 +- demo/webapp/{level1.html => policy.html} | 9 +- demo/webapp/public/img/ddpg-traffic.png | Bin 0 -> 32222 bytes demo/webapp/public/img/level0.png | Bin 30964 -> 26321 bytes .../js/{traffic => ddpg-traffic}/index.js | 23 +- demo/webapp/public/js/ddpg/ddpg.js | 12 +- demo/webapp/public/js/ddpg/ddpg_agent.js | 78 +- demo/webapp/public/js/ddpg/index.js | 12 +- demo/webapp/public/js/ddpg/models.js | 20 + .../js/policy_monte_carlo/policy_agent.js | 2 +- .../webapp/public/js/q_table/q_table_agent.js | 2 +- .../actor-model-ddpg-traffic.json | 2 +- .../actor-model-ddpg-traffic.weights.bin | Bin 21640 -> 59656 bytes .../critic-model-ddpg-traffic.json | 2 +- .../critic-model-ddpg-traffic.weights.bin | Bin 22276 -> 60932 bytes demo/webapp/{level0.html => qtable.html} | 9 +- src/asset_manager.ts | 5 +- src/basic_motion_engine.ts | 3 + src/bot_motion_engine.ts | 1 + src/car.ts | 6 +- src/embedded.ts | 6 +- src/embedded/level/level_3.ts | 816 ++++++++++++++++++ src/level.ts | 2 +- src/metacar.ts | 39 +- src/world.ts | 18 + 27 files changed, 1118 insertions(+), 102 deletions(-) rename demo/webapp/{level2.html => ddpg-traffic.html} (55%) create mode 100644 demo/webapp/ddpg.html rename demo/webapp/{level1.html => policy.html} (88%) create mode 100644 demo/webapp/public/img/ddpg-traffic.png rename demo/webapp/public/js/{traffic => ddpg-traffic}/index.js (71%) rename demo/webapp/{level0.html => qtable.html} (89%) create mode 100644 src/embedded/level/level_3.ts diff --git a/demo/webapp/level2.html b/demo/webapp/ddpg-traffic.html similarity index 55% rename from demo/webapp/level2.html rename to demo/webapp/ddpg-traffic.html index d0c93ee..ff6108b 100644 --- a/demo/webapp/level2.html +++ b/demo/webapp/ddpg-traffic.html @@ -2,7 +2,7 @@ - Metacar: Full control + Metacar: Continous control in traffic @@ -24,22 +24,30 @@

Metacar

-

Current state (Lidar points)



+

+

Current state (Lidar points)




- You can use the arrow keys to control the car by yourself.

- - This level is not solved yet (working on it). However, You can contribute to the project by solving it and sharing your implementation!

+ The algorithm is based on the following papers: +

    +
  • Deep Deterministic Policy Gradients (DDPG): paper
  • +
  • Parameter Space Noise for Exploration: paper
  • +
  • Prioritized Experience Replay: paper
  • +
+
+ You can use the arrow keys to control the car by yourself.

The motion control is based on two continuous values for the throttle and steering angle of the car.

The left window gives you an overview of what the autonomous vehicle (in red) sees.

+ + You can find the code of this demo here and here.

+
- @@ -48,11 +56,11 @@

Current state (Lidar points)



- - - - - - + + + + + + diff --git a/demo/webapp/ddpg.html b/demo/webapp/ddpg.html new file mode 100644 index 0000000..1fa1e0a --- /dev/null +++ b/demo/webapp/ddpg.html @@ -0,0 +1,66 @@ + + + + + Metacar: Continous control + + + + + + +
+
+

Metacar

+ +
+
+ +
+ +
+
+ +
+

+

Current state (Lidar points)




+
+

+ The algorithm is based on the following papers: +

    +
  • Deep Deterministic Policy Gradients (DDPG): paper
  • +
  • Parameter Space Noise for Exploration: paper
  • +
  • Prioritized Experience Replay: paper
  • +
+
+ You can use the arrow keys to control the car by yourself.

+ + The motion control is based on two continuous values for the throttle and steering angle of the car. + +

The left window gives you an overview of what the autonomous vehicle (in red) sees.

+ + You can find the code of this demo here. +

+
+ +
+
+ + + + + + + + + + + + + + + + + diff --git a/demo/webapp/index.html b/demo/webapp/index.html index cec0729..09c30eb 100644 --- a/demo/webapp/index.html +++ b/demo/webapp/index.html @@ -37,45 +37,72 @@

Check out examples of algorithms created with metacar




- -