Merge join changes to make indexing work. (#3)

* Use STACObjectType to identifiy objects

* Use STACObjectType to identifiy objects

* commit a run config

* process catalog is an list..

* add runconfig

* Use STACObjectType to identifiy objects

* Added NOTES_DEV.md
fixed some errors in stack for testing.

* Added NOTES_DEV.md
fixed some errors in stack for testing.

* fix some stac data
have collection, why do we search for it?

* Update NOTES_DEV.md

* Update NOTES_DEV.md

* Update walkstac.py

* add jsonld output

* update jsonlds

* generate sitemap

* separate 3 datasets; fix titles and descriptions; auto clear output (#6)

* move sitemaps

* fix ids

* update

* update (#9)

* remove output

* Yl dv demo (#10)

* update

* update

* update

* fix geoshape

* Resolve bounding box (#13)

* update 3 repos

* add def replace

* fix bbox

* fix intervals

* fix intervals

* fix schema

* fix schema

* download github repos

* Update walkstac.py

* update

* add workflow and requirement

* Update requirements.txt

* Update generate_json_files.yml

* Update walkstac.py

* Update generate_json_files.yml

* Update generate_json_files.yml

* add token

* Update generate_json_files.yml

* Update generate_json_files.yml

* Automated commit: Generate folder

* Update generate_json_files.yml

* Update generate_json_files.yml

* add titile

* Update generate_json_files.yml

* Automated commit: Update data folder

* Automated commit: Update data folder

* Automated commit: Update data folder

* Automated commit: Update data folder

* Automated commit: Update data folder

* Automated commit: Update data folder

* change keywords to array

* Update generate_json_files.yml

* Automated commit: Update data folder

* Automated commit: Update data folder

---------

Co-authored-by: Ya-Lan Yang <[email protected]>
Co-authored-by: Ya-Lan Yang <[email protected]>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/generate_json_files.yml

@@ -0,0 +1,59 @@
+name: Update sitemap and jsonlds
+
+on:
+  push:
+    branches: [ yl_dv ]
+  schedule:
+    # Runs at 00:00 UTC every day
+    - cron: '0 0 * * *'
+
+jobs:
+  update-sitemap-and-jsonlds:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+        ref: yl_dv  # Check out the `yl_dv` branch
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'  # Specify the Python version you need
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Clean up the target folder (if exists)
+      run: |
+        if [ -d "data/output" ]; then
+          rm -rf data/output/*
+        else
+          mkdir -p data/output
+        fi
+
+    - name: Run main.py with configfile
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        python main.py --configfile ./data/challenge/catalog-yl.json
+
+    - name: Stage changes in the data folder
+      run: |
+        git add data  # Stage only the `data` folder
+
+    - name: Commit and push changes if any
+      run: |
+        if git diff --cached --quiet; then
+          echo "No changes to commit."
+        else
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git commit -m "Automated commit: Update data folder"
+          git push origin yl_dv
+        fi
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

archive/spatial.py

@@ -18,14 +18,9 @@
 
 
 def sdo_box(bbox, cells):
-
     doc = {}
-    geo_a = []
-
-    geo = {}
-    geo["@type"] = "GeoShape"
-    geo["box"] = bbox
-    geo_a.append(geo)
+    additional_property = []
+    geo = {"@type": "GeoShape", "box": bbox}
 
     for c in cells:
         geos2 = {}
@@ -34,10 +29,10 @@ def sdo_box(bbox, cells):
         geos2["name"] = "s2Level13"
         geos2["description"] = "S2 cell at level 13"
         geos2["value"] = c
-        geo_a.append(geos2)
-
+        additional_property.append(geos2)
 
     doc['@type'] = 'Place'
-    doc['geo'] = geo_a
+    doc['geo'] = geo
+    doc['additionalProperty'] = additional_property
 
     return doc

data/challenge/NOTES_DEV.md

@@ -0,0 +1,62 @@
+# Notes on fixing issues
+
+# dates
+pending is not a date
+
+## href
+arrays and null is not a valid href
+changed to a valid empty href.
+
+{
+      "rel": "item",
+      "href": [],
+      "type": "text/html",
+      "title": "Link for Model Code"
+    }
+
+https://github.com/earthcube/stacIndexer/blob/yl/data/challenge/neon4cast-stac/forecasts/models/model_items/GLEON_physics.json
+```
+
+should not generate an asset, or may point to example.com
+
+```
+{
+      "rel": "item",
+      "href": "https://github.com/radiantearth",
+      "type": "text/html",
+      "title": "Link for Model Code"
+    }
+## hrefs cannot be arrays
+https://github.com/earthcube/stacIndexer/blob/8a64765c652a85438c54a5b773d0e5a09571ad30/data/challenge/neon4cast-stac/summaries/Aquatics/Daily_Chlorophyll_a/collection.json#L281
+'/Users/valentin/development/dev_earthcube/stacIndexer/data/challenge/neon4cast-stac/summaries/Aquatics/Daily_Chlorophyll_a/collection.json'
+
+['s3://anonymous@bio230014-bucket01/challenges/forecasts/parquet/project_id=neon4cast/duration=P1D/variable=chla?endpoint_override=sdsc.osn.xsede.org', 's3://anonymous@bio230014-bucket01/challenges/forecasts/parquet/project_id=usgsrc4cast/duration=P1D/variable=chla?endpoint_override=sdsc.osn.xsede.org']
+
+
+## bbox should not be lists
+
+"bbox": [
+
+      -156.6194,
+      71.2824,
+      -66.7987,
+      71.2824
+
+  ],
+
+(used two replace in files (
+1:
+"bbox": [
+    [
+
+2:
+
+))
+
+the bbox should be 
+"bbox": [
+      -156.6194,
+      71.2824,
+      -66.7987,
+      71.2824
+  ],

data/challenge/catalog-yl.json

@@ -1,38 +1,38 @@
 {
-  "type": "Catalog",
-  "id": "eco4cast-stac",
-  "title": "Ecological Forecasting Initiative Forecasting Challenges Catalog",
-  "description": "A STAC (Spatiotemporal Asset Catalog) describing forecasts and forecast scores for ecological forecasting challenges",
-  "stac_version": "1.0.0",
-  "conformsTo": "conformsTo",
-  "links": [
-    {
-      "rel": "self",
-      "type": "application/json",
-      "href": "catalog.json"
-    },
-    {
-      "rel": "root",
-      "type": "application/json",
-      "href": "catalog.json"
-    },
-    {
-      "rel": "child",
-      "type": "application/json",
-      "title": "neon4cast",
-      "href": "neon4cast-stac/catalog.json"
-    },
-    {
-      "rel": "child",
-      "type": "application/json",
-      "title": "vera4cast-LTREB-reservoirs",
-      "href": "vera4cast-stac/catalog.json"
-    },
+    "type": "Catalog",
+    "id": "eco4cast-stac",
+    "title": "Ecological Forecasting Initiative Forecasting Challenges Catalog",
+    "description": "A STAC (Spatiotemporal Asset Catalog) describing forecasts and forecast scores for ecological forecasting challenges",
+    "stac_version": "1.0.0",
+    "conformsTo": "conformsTo",
+    "links": [
         {
-      "rel": "child",
-      "type": "application/json",
-      "title": "vera4cast-eco4cast",
-      "href": "usgsrc4cast-stac/catalog.json"
-    }
-  ]
-}
+            "rel": "self",
+            "type": "application/json",
+            "href": "catalog.json"
+        },
+        {
+            "rel": "root",
+            "type": "application/json",
+            "href": "catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "neon4cast",
+            "href": "neon4cast-stac/catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "vera4cast-LTREB-reservoirs",
+            "href": "vera4cast-stac/catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "vera4cast-eco4cast",
+            "href": "usgsrc4cast-stac/catalog.json"
+        }
+    ]
+}

data/challenge/catalog.json

@@ -1,38 +1,38 @@
 {
-  "type": "Catalog",
-  "id": "eco4cast-stac",
-  "title": "Ecological Forecasting Initiative Forecasting Challenges Catalog",
-  "description": "A STAC (Spatiotemporal Asset Catalog) describing forecasts and forecast scores for ecological forecasting challenges",
-  "stac_version": "1.0.0",
-  "conformsTo": "conformsTo",
-  "links": [
-    {
-      "rel": "self",
-      "type": "application/json",
-      "href": "catalog.json"
-    },
-    {
-      "rel": "root",
-      "type": "application/json",
-      "href": "catalog.json"
-    },
-    {
-      "rel": "child",
-      "type": "application/json",
-      "title": "neon4cast",
-      "href": "https://raw.githubusercontent.com/eco4cast/neon4cast-ci/main/catalog/catalog.json"
-    },
-    {
-      "rel": "child",
-      "type": "application/json",
-      "title": "vera4cast",
-      "href": "https://raw.githubusercontent.com/LTREB-reservoirs/vera4cast/main/catalog/catalog.json"
-    },
+    "type": "Catalog",
+    "id": "eco4cast-stac",
+    "title": "Ecological Forecasting Initiative Forecasting Challenges Catalog",
+    "description": "A STAC (Spatiotemporal Asset Catalog) describing forecasts and forecast scores for ecological forecasting challenges",
+    "stac_version": "1.0.0",
+    "conformsTo": "conformsTo",
+    "links": [
         {
-      "rel": "child",
-      "type": "application/json",
-      "title": "vera4cast",
-      "href": "https://raw.githubusercontent.com/eco4cast/usgsrc4cast-ci/main/catalog/catalog.json"
-    }
-  ]
-}
+            "rel": "self",
+            "type": "application/json",
+            "href": "catalog.json"
+        },
+        {
+            "rel": "root",
+            "type": "application/json",
+            "href": "catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "neon4cast",
+            "href": "https://raw.githubusercontent.com/eco4cast/neon4cast-ci/main/catalog/catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "vera4cast",
+            "href": "https://raw.githubusercontent.com/LTREB-reservoirs/vera4cast/main/catalog/catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "vera4cast",
+            "href": "https://raw.githubusercontent.com/eco4cast/usgsrc4cast-ci/main/catalog/catalog.json"
+        }
+    ]
+}