simplify example folder skipping

dlt-hub · Mar 25, 2024 · b2af583 · b2af583
1 parent 1462382
commit b2af583
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 18 deletions.
diff --git a/docs/tools/prepare_examples_tests.py b/docs/tools/prepare_examples_tests.py
@@ -9,7 +9,7 @@
 EXAMPLES_DIR = "../examples"
 
 # settings
-SKIP_FOLDERS = ["archive", ".dlt", "__pycache__", "_storage", "local_cache"]
+SKIP_FOLDERS = ["archive", ".", "_", "local_cache"]
 
 # the entry point for the script
 MAIN_CLAUSE = 'if __name__ == "__main__":'
@@ -36,10 +36,11 @@
 
     count = 0
     for example in next(os.walk(EXAMPLES_DIR))[1]:
-        if example in SKIP_FOLDERS:
+        # skip some
+        if any(map(lambda skip: example.startswith(skip), SKIP_FOLDERS)):
             continue
-        count += 1
 
+        count += 1
         example_file = f"{EXAMPLES_DIR}/{example}/{example}.py"
         test_example_file = f"{EXAMPLES_DIR}/{example}/test_{example}.py"
 

diff --git a/docs/website/tools/preprocess_docs.js b/docs/website/tools/preprocess_docs.js
@@ -17,6 +17,7 @@ const SNIPPETS_FILE_SUFFIX = "-snippets.py"
 // examples settings
 const EXAMPLES_DESTINATION_DIR = `./${MD_TARGET_DIR}examples/`;
 const EXAMPLES_SOURCE_DIR = "../examples/";
+const EXAMPLES_EXCLUSIONS = [".", "_", "archive", "local_cache"]
 
 // markers
 const DLT_MARKER = "@@@DLT";
@@ -262,22 +263,16 @@ function syncExamples() {
 
   let count = 0;
   for (const exampleDir of listDirsSync(EXAMPLES_SOURCE_DIR)) {
-    if (exampleDir.includes("archive")) {
-      continue;
-    }
-    if (exampleDir.includes(".dlt")) {
-      continue;
-    }
-    if (exampleDir.includes("__pycache__")) {
-      continue;
-    }
-    if (exampleDir.includes("_storage")) {
-      continue;
-    }
-    if (exampleDir.includes("local_cache")) {
-      continue;
-    }
+
     const exampleName = exampleDir.split("/").slice(-1)[0];
+
+    // exclude some folders
+    for (const exclusion of EXAMPLES_EXCLUSIONS) {
+      if (exampleName.startsWith(exclusion)) {
+        continue;
+      }
+    }
+
     const exampleFile = `${EXAMPLES_SOURCE_DIR}${exampleName}/${exampleName}.py`;
     const targetFileName = `${EXAMPLES_DESTINATION_DIR}/${exampleName}.md`;
     const lines = fs.readFileSync(exampleFile, 'utf8').split(/\r?\n/);