araison
/
explaining_framework
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Spitting configs to differents GPUs stack

This commit is contained in:
araison 2023-01-09 18:56:42 +01:00
parent b1093eb528
commit d1c2565003
1 changed files with 53 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
explaining_framework/utils/config_gen.py
View File

@ -1,4 +1,6 @@
import glob
import os import os
import shutil
from explaining_framework.utils.io import write_yaml from explaining_framework.utils.io import write_yaml
from torch_geometric.data.makedirs import makedirs from torch_geometric.data.makedirs import makedirs
@ -91,38 +93,54 @@ if "__main__" == __name__:
"SCGNN", "SCGNN",
] ]
for dataset_name in DATASET: # for dataset_name in DATASET:
dataset = load_pyg_dataset(name=dataset_name, dataset_dir="/tmp/") # try:
for chunk in chunkizing_list(list(range(len(dataset))), 300): # dataset = load_pyg_dataset(name=dataset_name, dataset_dir="/tmp/")
for model_kind in ["best", "worst"]: # except Exception as e:
for explainer_name in EXPLAINER: # print(e)
explaining_cfg = {} # continue
# explaining_cfg['adjust']['strategy']= 'rpns' # for chunk in chunkizing_list(list(range(len(dataset))), 1000):
# explaining_cfg['attack']['name']= 'all' # for model_kind in ["best", "worst"]:
explaining_cfg[ # for explainer_name in EXPLAINER:
"cfg_dest" # explaining_cfg = {}
] = f"dataset={dataset_name}-model={model_kind}=explainer={explainer_name}-chunk=[{chunk[0]},{chunk[-1]}]" # # explaining_cfg['adjust']['strategy']= 'rpns'
explaining_cfg["dataset"] = {} # # explaining_cfg['attack']['name']= 'all'
explaining_cfg["dataset"]["name"] = dataset_name # explaining_cfg[
explaining_cfg["dataset"]["item"] = chunk # "cfg_dest"
# explaining_cfg['explainer']['cfg']= 'default' # ] = f"dataset={dataset_name}-model={model_kind}=explainer={explainer_name}-chunk=[{chunk[0]},{chunk[-1]}]"
explaining_cfg["explainer"] = {} # explaining_cfg["dataset"] = {}
explaining_cfg["explainer"]["name"] = explainer_name # explaining_cfg["dataset"]["name"] = dataset_name
explaining_cfg["explanation_type"] = "phenomenon" # explaining_cfg["dataset"]["item"] = chunk
# explaining_cfg['metrics']['accuracy']['name']='all' # # explaining_cfg['explainer']['cfg']= 'default'
# explaining_cfg['metrics']['fidelity']['name']='all' # explaining_cfg["explainer"] = {}
# explaining_cfg['metrics']['sparsity']['name']='all' # explaining_cfg["explainer"]["name"] = explainer_name
explaining_cfg["model"] = {} # explaining_cfg["explanation_type"] = "phenomenon"
explaining_cfg["model"]["ckpt"] = model_kind # # explaining_cfg['metrics']['accuracy']['name']='all'
explaining_cfg["model"][ # # explaining_cfg['metrics']['fidelity']['name']='all'
"path" # # explaining_cfg['metrics']['sparsity']['name']='all'
] = "/home/SIC/araison/test_ggym/pytorch_geometric/graphgym/results" # explaining_cfg["model"] = {}
# explaining_cfg['out_dir']='./explanation' # explaining_cfg["model"]["ckpt"] = model_kind
# explaining_cfg['print']='both' # explaining_cfg["model"][
# explaining_cfg['threshold']['config']['type']='all' # "path"
# explaining_cfg['threshold']['value']['hard']=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] # ] = "/home/SIC/araison/test_ggym/pytorch_geometric/graphgym/results"
# explaining_cfg['threshold']['value']['topk']=[2, 3, 5, 10, 20, 30, 50] # # explaining_cfg['out_dir']='./explanation'
write_yaml( # # explaining_cfg['print']='both'
explaining_cfg, # # explaining_cfg['threshold']['config']['type']='all'
explaining_folder + "/" + explaining_cfg["cfg_dest"] + ".yaml", # # explaining_cfg['threshold']['value']['hard']=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
) # # explaining_cfg['threshold']['value']['topk']=[2, 3, 5, 10, 20, 30, 50]
# PATH = os.path.join(
# explaining_folder + "/" + explaining_cfg["cfg_dest"] + ".yaml",
# )
# if os.path.exists(PATH):
# continue
# else:
# write_yaml(explaining_cfg, PATH)
configs = [path for path in glob.glob(os.path.join(explaining_folder, "*.yaml"))]
for index, config_chunk in enumerate(
chunkizing_list(configs, int(len(configs) / 5))
):
PATH_ = os.path.join(explaining_folder, f"gpu={index}")
makedirs(PATH_)
for path in config_chunk:
filename = os.path.basename(path)
shutil.copy2(path, os.path.join(PATH_, filename))