Fix

csis_scraper/scrape/scrape.py

@@ -641,7 +641,11 @@ def traverse_papers(self, page_url, volume_no):
             new_title = str(title).replace(" ", "_")
             source_file = os.path.join(self.pdf_directory, pdf_id) + ".pdf"
             target_file = os.path.join(self.pdf_directory, new_title) + ".pdf"
+            target_path_list = target_file.split("/")
+            if not re.search(r"volume_\d+", target_path_list[-2]):
+                target_path_list[-2] = target_path_list[-2] + "_" + target_path_list[-1]
+                target_path_list = target_path_list[:-1]
+            target_file = os.path.join(*target_path_list)
             os.rename(source_file, target_file)
-
             scraped_docs_list.append(scraped)
         return scraped_docs_list

main.py

@@ -15,6 +15,7 @@
 import yaml
 import shutil
 import tqdm
+import re
 
 
 def tuple_type(strings):
@@ -34,6 +35,16 @@ def return_config(path):
         return yaml.safe_load(file)
 
 
+def prepare_ttl_path(ttl_dir, article):
+    ttl_path_split = os.path.join(ttl_dir, article.title).split("/")
+    if not re.search(r"volume_\d+", ttl_path_split[-2]):
+        ttl_path_split[-2] = ttl_path_split[-2] + "_" + ttl_path_split[-1]
+        ttl_path_split = ttl_path_split[:-1]
+    ttl_filename = os.path.join(*ttl_path_split) + ".ttl"
+    ttl_filename = ttl_filename.replace(" ", "_")
+    return ttl_filename
+
+
 def scrape_scpe(scpe_issues_to_scrape):
     print("Processing SCPE archives...")
     logging.basicConfig(
@@ -66,8 +77,7 @@ def scrape_scpe(scpe_issues_to_scrape):
         if not os.path.exists(ttl_dir):
             os.makedirs(ttl_dir)
 
-        ttl_filename = os.path.join(ttl_dir, os.path.basename(article.title + ".ttl"))
-        ttl_filename = ttl_filename.replace(" ", "_")
+        ttl_filename = prepare_ttl_path(ttl_dir, article)
 
         with open(ttl_filename, "w") as file:
             file.write(g)
@@ -100,11 +110,7 @@ def scrape_csis(csis_volumes_to_scrape, args):
             ttl_dir = f"./output/ttls/csis/volume_{article.volume}"
             if not os.path.exists(ttl_dir):
                 os.mkdir(ttl_dir)
-
-            ttl_filename = os.path.join(
-                ttl_dir, os.path.basename(article.title + ".ttl")
-            )
-            ttl_filename = ttl_filename.replace(" ", "_")
+            ttl_filename = prepare_ttl_path(ttl_dir, article)
 
             with open(ttl_filename, "w") as file:
                 file.write(g)