update processing

fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/final_post_processing.py

@@ -19,7 +19,7 @@
         del c["tstamp"]
         del c["user_id"]
 
-    np.random.seed(42)
+    np.random.seed(44)
     np.random.shuffle(convs)
 
     convs = convs[:args.number]

fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/process_all.sh

@@ -0,0 +1,17 @@
+export BASE=clean_conv_20230809_1.5M_pii
+export SCALE=1
+
+# filter words
+python3 filter_bad_conv.py --in $BASE.json
+
+# Clean up some fileds (e.g., timestamps)
+python3 final_post_processing.py --in $BASE.s1.json
+
+# upload to hf
+python3 upload_hf_dataset.py --in $BASE.s1.s2.json
+
+# Make another version with openai moderation tag
+python3 merge_oai_tag.py --in $BASE.s1.s2.json
+
+# Make visualizations
+python3 compute_stats.py --in $BASE.s1.json --scale $SCALE