dpgen run stuck

[kluophysics]

After new installation of dpgen, I tried to run the program as the following. It was running okay before the new installation. But now it got stuck at the first step.

The file inside the scratch dir is empty!

The machine.json is

{
"api_version": "1.0",
"train" :[
{
"command": "/.conda/envs/deepmd/bin/dp",
"machine": {
"batch_type": "slurm",
"context_type": "SSHContext",
"local_root" : "./",
"remote_root": "/public/home/kluo/work/calc/dpgen/scratch",
"remote_profile":{
"hostname": "localhost",
"username": "kluo"
}
},
"resources": {
"number_node": 1,
"cpu_per_node": 8,
"gpu_per_node": 0,
"queue_name": "small",
"group_size": 1,
"source_list": ["/public/home/kluo/work/calc/dpgen/Al-liq-vapor/run/dp.env"]
}
}],
"model_devi":
[{
"command": "/.conda/envs/deepmd/bin/lmp",
"machine": {
"batch_type": "slurm",
"context_type": "SSHContext",
"local_root" : "./",
"remote_root": "/public/home/kluo/work/calc/dpgen/scratch",
"remote_profile":{
"hostname": "localhost",
"username": "kluo"
}
},
"resources": {
"number_node": 1,
"cpu_per_node": 36,
"gpu_per_node": 0,
"queue_name": "medium",
"group_size": 20,
"source_list": ["/public/home/kluo/work/calc/dpgen/Al-liq-vapor/run/lmp.env"]
}
}],
"fp":
[{
"command": "mpirun -np $SLURM_NTASKS vasp_std",
"machine": {
"batch_type": "slurm",
"context_type": "SSHContext",
"local_root" : "./",
"remote_root": "/public/home/kluo/work/calc/dpgen/scratch",
"remote_profile":{
"hostname": "localhost",
"username": "kluo"
}
},
"resources": {
"number_node": 1,
"cpu_per_node": 36,
"gpu_per_node": 0,
"queue_name": "medium",
"group_size": 10,
"custom_flags": ["#SBATCH --time=20-00:00:00", "#SBATCH --job-name=vasp_dpgen"],
"module_purge": true,
"module_list": ["mpi/intelmpi/2017.4.239", "compiler/intel/intel-compiler-2017.5.239", "apps/vasp/5.4.1"],
"source_list": [],
"_comment": "if use 2 nodes, then with #SBATCH --exclude=node5,node20. otherwise, do not include it"
}
}]
}

the param.json is

{
"type_map": ["Fe"],
"mass_map": [55.847],
"use_ele_temp": 1,

"init_data_prefix":	"../sys",

"init_data_sys":	[
"train_data"
],
"init_multi_systems": true,
"sys_configs_prefix": "../init",
"sys_configs":	[
["hcp/fe.hcp.03x03x03/01.scale_pert/sys-0054/scale*/0000*/POSCAR"],
    ["bcc/fe.bcc.03x03x03/01.scale_pert/sys-0054/scale*/0000*/POSCAR"]
],
"_comment":		" 00.train ",
"numb_models":	4,
"default_training_param" : {
"model":{
"_comment": " model parameters",
"type_map":["Fe"],
"descriptor":{
"type":			"se_a",
"sel":			"auto",
"rcut_smth":		0.80,
"rcut":			5.50,
"neuron":		[25, 50, 100],
"resnet_dt":		false,
"axis_neuron":		12,
"seed":			1
},
"fitting_net":{
"neuron":		[240, 240, 240],
"resnet_dt":		true,
    "numb_fparam": 1,
"seed": 1
}
},
"learning_rate":{
"type":			"exp",
"start_lr":		0.001,
    "stop_lr":              1e-8,
"decay_steps":		2000
},
"loss":{
"start_pref_e":		0.2,
"limit_pref_e":		1,
"start_pref_f":		1000,
"limit_pref_f":		1,
"start_pref_v":		0.2,
"limit_pref_v":		1.0
},
"training":{
	"set_prefix":	"set",
	"stop_batch":	100000,
	"batch_size": 4,
	"disp_file":		"lcurve.out",
	"disp_freq":		2000,
	"numb_test":		4,
	"save_freq":		10000,
	"save_ckpt":		"model.ckpt",
	"disp_training": true,
	"time_training": true,
	"profiling": false,
	"profiling_file": "timeline.json"
}
},

"_comment":		" 01.model_devi ",
"_comment": "model_devi_skip: the first x of the recorded frames",
"model_devi_dt":		0.0010,
"model_devi_skip":		0,
"model_devi_f_trust_lo":	0.30,
"model_devi_f_trust_hi":	0.50,
"model_devi_clean_traj":	true,
"model_devi_jobs":	[ 

],

"_comment":		"02.fp ",
"fp_style":		"vasp",
"shuffle_poscar":	false,
"fp_task_max":	400,
"fp_task_min":	20,
"fp_pp_path":	"./",
"fp_pp_files":	["POTCAR_14e_Fe"],
"fp_incar":         "INCAR_scf",
"_comment":		" that's all "

}

During the run, a .tgz file was generated somehow, which wasn't seen before.

$ ls 00.train/
000 001 002 003 data.init eacfa1fffd6cec401a47d6ca4318895982d926ad.tgz

Any idea of how the stuck happens?Thanks!

Kai

[kluophysics]

I figured it out the conda environment (default source inside .bashrc file) somehow interfere with the ssh connection. After disabling it, now the jobs starts.

[AnguseZhang]

This is also the first time I've seen such problem.

[AnguseZhang]

If you run all jobs on a cluster, there's no need to use SSH context. See #722

[kluophysics]

I will take out the SSHcontex.