Update FoS query, fix path, update schema

georgetown-cset · Aug 30, 2024 · 1f34223 · 1f34223
1 parent feb09ad
commit 1f34223
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 3 deletions.
diff --git a/schemas/repos_with_full_meta_raw.json b/schemas/repos_with_full_meta_raw.json
@@ -2400,6 +2400,18 @@
             "mode": "NULLABLE",
             "name": "secret_scanning_validity_checks",
             "type": "RECORD"
+          },
+          {
+            "fields": [
+              {
+                "mode": "NULLABLE",
+                "name": "status",
+                "type": "STRING"
+              }
+            ],
+            "name": "secret_scanning_non_provider_patterns",
+            "mode": "NULLABLE",
+            "type": "RECORD"
           }
         ],
         "mode": "NULLABLE",

diff --git a/scripts/scrape_additional_metadata.sh b/scripts/scrape_additional_metadata.sh
@@ -9,5 +9,5 @@ done_file="scrape_additional_metadata"
 cd $working_dir
 gsutil rm "gs://${data_bucket}/${production_dataset}/done_files/${done_file}"
 PYTHONPATH='.' python3 scripts/retrieve_repo_metadata.py curr_repos_filled.jsonl curr_repos_final.jsonl
-gsutil cp curr_repos_final.jsonl "gs://${data_bucket}/${tmp_dir}" && touch $done_file
+gsutil cp curr_repos_final.jsonl "gs://${data_bucket}/${tmp_dir}/" && touch $done_file
 gsutil cp $done_file "gs://${data_bucket}/${production_dataset}/done_files/"
diff --git a/sql/top_level1_fields.sql b/sql/top_level1_fields.sql
@@ -3,7 +3,7 @@ WITH
 field_name_scores AS (
   SELECT
     merged_id,
-    name,
+    field.name,
     field.score AS score
   FROM
     fields_of_study_v2.field_scores
@@ -12,7 +12,7 @@ field_name_scores AS (
   LEFT JOIN
     fields_of_study_v2.field_meta
     ON
-      field_id = field.id
+      field_meta.name = field.name
   WHERE
     (level = 1)),