From d1c2565003733ab220ee654431f02b775fe06b44 Mon Sep 17 00:00:00 2001
From: araison <adrien.raison@xlim.fr>
Date: Mon, 9 Jan 2023 18:56:42 +0100
Subject: [PATCH] Spitting configs to differents GPUs stack

---
 explaining_framework/utils/config_gen.py | 88 ++++++++++++++----------
 1 file changed, 53 insertions(+), 35 deletions(-)

diff --git a/explaining_framework/utils/config_gen.py b/explaining_framework/utils/config_gen.py
index 0fdd6a8..1180383 100644
--- a/explaining_framework/utils/config_gen.py
+++ b/explaining_framework/utils/config_gen.py
@@ -1,4 +1,6 @@
+import glob
 import os
+import shutil
 
 from explaining_framework.utils.io import write_yaml
 from torch_geometric.data.makedirs import makedirs
@@ -91,38 +93,54 @@ if "__main__" == __name__:
         "SCGNN",
     ]
 
-    for dataset_name in DATASET:
-        dataset = load_pyg_dataset(name=dataset_name, dataset_dir="/tmp/")
-        for chunk in chunkizing_list(list(range(len(dataset))), 300):
-            for model_kind in ["best", "worst"]:
-                for explainer_name in EXPLAINER:
-                    explaining_cfg = {}
-                    # explaining_cfg['adjust']['strategy']= 'rpns'
-                    # explaining_cfg['attack']['name']= 'all'
-                    explaining_cfg[
-                        "cfg_dest"
-                    ] = f"dataset={dataset_name}-model={model_kind}=explainer={explainer_name}-chunk=[{chunk[0]},{chunk[-1]}]"
-                    explaining_cfg["dataset"] = {}
-                    explaining_cfg["dataset"]["name"] = dataset_name
-                    explaining_cfg["dataset"]["item"] = chunk
-                    # explaining_cfg['explainer']['cfg']= 'default'
-                    explaining_cfg["explainer"] = {}
-                    explaining_cfg["explainer"]["name"] = explainer_name
-                    explaining_cfg["explanation_type"] = "phenomenon"
-                    # explaining_cfg['metrics']['accuracy']['name']='all'
-                    # explaining_cfg['metrics']['fidelity']['name']='all'
-                    # explaining_cfg['metrics']['sparsity']['name']='all'
-                    explaining_cfg["model"] = {}
-                    explaining_cfg["model"]["ckpt"] = model_kind
-                    explaining_cfg["model"][
-                        "path"
-                    ] = "/home/SIC/araison/test_ggym/pytorch_geometric/graphgym/results"
-                    # explaining_cfg['out_dir']='./explanation'
-                    # explaining_cfg['print']='both'
-                    # explaining_cfg['threshold']['config']['type']='all'
-                    # explaining_cfg['threshold']['value']['hard']=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
-                    # explaining_cfg['threshold']['value']['topk']=[2, 3, 5, 10, 20, 30, 50]
-                    write_yaml(
-                        explaining_cfg,
-                        explaining_folder + "/" + explaining_cfg["cfg_dest"] + ".yaml",
-                    )
+    # for dataset_name in DATASET:
+    # try:
+    # dataset = load_pyg_dataset(name=dataset_name, dataset_dir="/tmp/")
+    # except Exception as e:
+    # print(e)
+    # continue
+    # for chunk in chunkizing_list(list(range(len(dataset))), 1000):
+    # for model_kind in ["best", "worst"]:
+    # for explainer_name in EXPLAINER:
+    # explaining_cfg = {}
+    # # explaining_cfg['adjust']['strategy']= 'rpns'
+    # # explaining_cfg['attack']['name']= 'all'
+    # explaining_cfg[
+    # "cfg_dest"
+    # ] = f"dataset={dataset_name}-model={model_kind}=explainer={explainer_name}-chunk=[{chunk[0]},{chunk[-1]}]"
+    # explaining_cfg["dataset"] = {}
+    # explaining_cfg["dataset"]["name"] = dataset_name
+    # explaining_cfg["dataset"]["item"] = chunk
+    # # explaining_cfg['explainer']['cfg']= 'default'
+    # explaining_cfg["explainer"] = {}
+    # explaining_cfg["explainer"]["name"] = explainer_name
+    # explaining_cfg["explanation_type"] = "phenomenon"
+    # # explaining_cfg['metrics']['accuracy']['name']='all'
+    # # explaining_cfg['metrics']['fidelity']['name']='all'
+    # # explaining_cfg['metrics']['sparsity']['name']='all'
+    # explaining_cfg["model"] = {}
+    # explaining_cfg["model"]["ckpt"] = model_kind
+    # explaining_cfg["model"][
+    # "path"
+    # ] = "/home/SIC/araison/test_ggym/pytorch_geometric/graphgym/results"
+    # # explaining_cfg['out_dir']='./explanation'
+    # # explaining_cfg['print']='both'
+    # # explaining_cfg['threshold']['config']['type']='all'
+    # # explaining_cfg['threshold']['value']['hard']=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
+    # # explaining_cfg['threshold']['value']['topk']=[2, 3, 5, 10, 20, 30, 50]
+    # PATH = os.path.join(
+    # explaining_folder + "/" + explaining_cfg["cfg_dest"] + ".yaml",
+    # )
+    # if os.path.exists(PATH):
+    # continue
+    # else:
+    # write_yaml(explaining_cfg, PATH)
+    configs = [path for path in glob.glob(os.path.join(explaining_folder, "*.yaml"))]
+    for index, config_chunk in enumerate(
+        chunkizing_list(configs, int(len(configs) / 5))
+    ):
+        PATH_ = os.path.join(explaining_folder, f"gpu={index}")
+        makedirs(PATH_)
+        for path in config_chunk:
+            filename = os.path.basename(path)
+            shutil.copy2(path, os.path.join(PATH_, filename))