Fixing bug and redoing all PPO experiments

experiments/mpsc/models/rl_models/quadrotor_3D/track/ppo/mpsf_sr_pen_1/seed_4077/logs/loss/approx_kl.log

@@ -1,101 +1,101 @@
 step,loss/approx_kl
-10000,0.028696737089194364
-20000,0.030057967298974593
-30000,0.025611055254315333
-40000,0.016911696968600155
-50000,0.030194475703562297
-60000,0.023369391355663537
-70000,0.021996175233895583
-80000,0.021863334556110205
-90000,0.03568383481663962
-100000,0.014697150342787305
-110000,0.03177917617528389
-120000,0.026181375654414297
-130000,0.021285173250362277
-140000,0.013811860776816806
-150000,0.026899692757676043
-160000,0.022368550905957817
-170000,0.017832595234115917
-180000,0.023721250379458072
-190000,0.01288926855971416
-200000,0.031091344806676108
-210000,0.03509402060881257
-220000,0.01624752441421151
-230000,0.028650374679515755
-240000,0.027777481405064468
-250000,0.029565240054701764
-260000,0.02017177523424228
-270000,0.02249487469283243
-280000,0.01951470755351086
-290000,0.021842470190798245
-300000,0.02333014807663858
-310000,0.021366345261534053
-320000,0.03473256565630436
-330000,0.02720941910520196
-340000,0.024917483360817037
-350000,0.016342141913870973
-360000,0.025328213737035792
-370000,0.024121754507844644
-380000,0.03186427508480847
-390000,0.028433764912188048
-400000,0.0318025677309682
-410000,0.008333341513449947
-420000,0.02042762814089656
-430000,0.029991648082310956
-440000,0.02331252739143868
-450000,0.025671916905169685
-460000,0.0232216097569714
-470000,0.02528135239457091
-480000,0.01746960976161063
-490000,0.03274932447820901
-500000,0.014156001945957541
-510000,0.02665379446310301
-520000,0.02191540490215023
-530000,0.028994616183141874
-540000,0.01702198113004367
-550000,0.028701075585559017
-560000,0.021187896436701215
-570000,0.019701794752230245
-580000,0.027341614430770277
-590000,0.02880933924267689
-600000,0.021775124035775664
-610000,0.022335819636161127
-620000,0.0111508471891284
-630000,0.030648241098970175
-640000,0.03295646039769053
-650000,0.02809145286058386
-660000,0.02844463465735317
-670000,0.019245932856574655
-680000,0.02468275399878621
-690000,0.02420532793427507
-700000,0.020767471892759203
-710000,0.02053832596478363
-720000,0.02562923206326862
-730000,0.03506784976149598
-740000,0.012499983888119461
-750000,0.030258858079711605
-760000,0.023030434176325797
-770000,0.027445856792231403
-780000,0.022826581044743454
-790000,0.032410532763848705
-800000,0.03811043140788874
-810000,0.018143621863176425
-820000,0.022016861910621327
-830000,0.0221443603746593
-840000,0.013442413043230775
-850000,0.017857752926647662
-860000,0.033521514510114994
-870000,0.029461463664968806
-880000,0.027809128692994522
-890000,0.02072050279627244
-900000,0.01958664289365212
-910000,0.024326797481626274
-920000,0.027207750050971896
-930000,0.02532225605100393
-940000,0.03086591030781468
-950000,0.026631653929750122
-960000,0.02141410972302159
-970000,0.024979121889919045
-980000,0.021551774131755037
-990000,0.021817117681105926
-1000000,0.027239296802630025
+10000,0.029208557703532277
+20000,0.032586767640896144
+30000,0.025630848994478584
+40000,0.018259687620835997
+50000,0.011078280257061124
+60000,0.01784560952801257
+70000,0.028387357271276413
+80000,0.014744772606839738
+90000,0.022554900233323374
+100000,0.03381081408200164
+110000,0.021157057660942273
+120000,0.012980545591562986
+130000,0.01448677368462086
+140000,0.01618971132362882
+150000,0.027147181704640387
+160000,0.03227456621825696
+170000,0.028502518093834327
+180000,0.03360094296125075
+190000,0.02035950571298599
+200000,0.030597594985738396
+210000,0.029150963341817254
+220000,0.021370276901870967
+230000,0.026677678882454835
+240000,0.02569064134731889
+250000,0.023797816317528486
+260000,0.03238879423588514
+270000,0.03242140817455947
+280000,0.029922109438727296
+290000,0.02502187010832131
+300000,0.025891695031896235
+310000,0.03266000323928893
+320000,0.020521501762171584
+330000,0.0321559206272165
+340000,0.03273177132941782
+350000,0.02074912659203013
+360000,0.021280657981211943
+370000,0.026237463330229128
+380000,0.024662876967340704
+390000,0.028879422973841433
+400000,0.03003662669410308
+410000,0.028840302008514607
+420000,0.027182075738285977
+430000,0.022763655552019674
+440000,0.031080836523324246
+450000,0.027716765382016693
+460000,0.03432357050478459
+470000,0.023513463124011955
+480000,0.015433562810843187
+490000,0.03728839297158022
+500000,0.02360844494154056
+510000,0.03617782692114512
+520000,0.029703678097575903
+530000,0.03617854948776463
+540000,0.03279251842759549
+550000,0.02574667798665663
+560000,0.03175092145490149
+570000,0.024243962081770105
+580000,0.03132888713541131
+590000,0.0228570561700811
+600000,0.0334127701818943
+610000,0.03893137550912798
+620000,0.03610405530780554
+630000,0.020595745866497362
+640000,0.024008269033705194
+650000,0.021521438083921866
+660000,0.030270544004937016
+670000,0.027262707939371474
+680000,0.02819207076293726
+690000,0.02912460401033362
+700000,0.027005492011085153
+710000,0.02883505852272113
+720000,0.01758278487250209
+730000,0.02539473500413199
+740000,0.023108853834370773
+750000,0.03146429948198299
+760000,0.02944188287171225
+770000,0.023764610554402075
+780000,0.015817845379933713
+790000,0.029224518127739428
+800000,0.03301882237816851
+810000,0.01593577017386754
+820000,0.01645373026840389
+830000,0.021518756107737624
+840000,0.02098657254440089
+850000,0.013631100362787644
+860000,0.021790609570840995
+870000,0.016325075551867487
+880000,0.031860177777707575
+890000,0.028470443344364566
+900000,0.020560097849617404
+910000,0.028620055193702382
+920000,0.020117931657781203
+930000,0.008297148657341798
+940000,0.028316706574211516
+950000,0.03018105545391639
+960000,0.017713127906123796
+970000,0.03288293123866121
+980000,0.014923745207488537
+990000,0.02158841568355759
+1000000,0.025475860014557834