Skip to content

Commit

Permalink
Improve STAC API asset and metadata implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
@m-mohr
m-mohr committed May 31, 2024
1 parent 6484f63 commit 158b6f4
Show file tree
Hide file tree
Showing 3 changed files with 188 additions and 29 deletions.
22 changes: 12 additions & 10 deletions src/api/collections.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,7 @@ export default class Data {
server.addEndpoint('get', '/collections/{collection_id}/queryables', this.getCollectionQueryables.bind(this));
server.addEndpoint('get', '/collections/{collection_id}/items', this.getCollectionItems.bind(this));
server.addEndpoint('get', '/collections/{collection_id}/items/{item_id}', this.getCollectionItemById.bind(this));
if (this.context.stacAssetDownload) {
server.addEndpoint('get', ['/assets/{asset_id}', '/assets/*'], this.getAssetById.bind(this));
}
server.addEndpoint('get', ['/assets/{asset_id}', '/assets/*'], this.getAssetById.bind(this));
server.addEndpoint('get', ['/thumbnails/{asset_id}', '/thumbnails/*'], this.getThumbnailById.bind(this));

const a = Date.now();
Expand Down Expand Up @@ -143,7 +141,7 @@ export default class Data {
id = req.params['*'].replace(/\/items$/, '');
}

const collection = this.catalog.getData(id);
const collection = this.catalog.getData(id, true);
if (collection === null) {
throw new Errors.CollectionNotFound();
}
Expand Down Expand Up @@ -241,7 +239,7 @@ export default class Data {
.catch(console.error);

// Convert to STAC
const features = items.map(item => this.catalog.convertImageToStac(item, id));
const features = items.map(item => this.catalog.convertImageToStac(item, collection));
// Add links
const links = [
{
Expand Down Expand Up @@ -299,7 +297,7 @@ export default class Data {
id = match[2];
}

const collection = this.catalog.getData(cid);
const collection = this.catalog.getData(cid, true);
if (collection === null) {
throw new Errors.CollectionNotFound();
}
Expand All @@ -316,7 +314,7 @@ export default class Data {
}
}

res.json(this.catalog.convertImageToStac(metadata, cid));
res.json(this.catalog.convertImageToStac(metadata, collection));
}

async getThumbnailById(req, res) {
Expand All @@ -339,7 +337,7 @@ export default class Data {
const img = this.ee.Image(id);
const geeURL = await new Promise((resolve, reject) => {
img.visualize(vis.band_vis).getThumbURL({
dimensions: 1000,
dimensions: 600,
crs: 'EPSG:3857',
format: 'png'
}, (geeUrl, err) => {
Expand All @@ -361,12 +359,16 @@ export default class Data {

async getAssetById(req, res) {
const id = req.params['*'];
const band = req.query.band || null;

const img = this.ee.Image(id);
let img = this.ee.Image(id);
if (band) {
img = img.select(band);
}
const crs = 'EPSG:4326';
const geeURL = await new Promise((resolve, reject) => {
img.getDownloadURL({
dimensions: 1000,
dimensions: this.context.stacAssetDownloadSize,
region: img.geometry(null, crs),
crs,
filePerBand: false,
Expand Down
193 changes: 175 additions & 18 deletions src/models/catalog.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,13 @@ import ItemStore from './itemstore.js';
import { Storage } from '@google-cloud/storage';
import API from '../utils/API.js';

const STAC_DATACUBE_EXTENSION = "https://stac-extensions.github.io/datacube/v2.2.0/schema.json";
const STAC_EXTENSIONS = {
cube: "https://stac-extensions.github.io/datacube/v2.2.0/schema.json",
eo: "https://stac-extensions.github.io/eo/v1.1.0/schema.json",
proj: "https://stac-extensions.github.io/projection/v1.1.0/schema.json",
raster: "https://stac-extensions.github.io/raster/v1.1.0/schema.json",
version: "https://stac-extensions.github.io/version/v1.0.0/schema.json"
};

// Rough auto mapping for common band names until GEE lists them.
// Optimized for Copernicus S2 data
Expand Down Expand Up @@ -52,6 +58,43 @@ const geeSchemaMapping = {
}
};

const geeDataTypeMapping = {
int: [
{
name: "int8",
min: -128,
max: 127
},
{
name: "uint8",
min: 0,
max: 255
},
{
name: "int16",
min: -32768,
max: 32767,
},
{
name: "uint16",
min: 0,
max: 65535,
},
{
name: "int32",
min: -2147483648,
max: 2147483647,
},
{
name: "uint32",
min: 0,
max: 4294967295,
}
],
float: "float32",
double: "float64"
};

export default class DataCatalog {

constructor(context) {
Expand Down Expand Up @@ -144,7 +187,7 @@ export default class DataCatalog {
return null;
}

const geeSchemas = collection.summaries['gee:schema'];
const geeSchemas = collection.summaries['gee:schema'] || [];
const jsonSchema = {
"$schema" : "https://json-schema.org/draft/2019-09/schema",
"$id" : API.getUrl(`/collections/${id}/queryables`),
Expand Down Expand Up @@ -238,6 +281,7 @@ export default class DataCatalog {
"system:time_start",
"system:time_end"
];
const extensions = new Set();

const id = img.properties["system:index"];
let geometry = null;
Expand All @@ -250,16 +294,28 @@ export default class DataCatalog {
bbox = Utils.geoJsonBbox(geometry, true);
}

const bands = img.bands.map(b => ({
name: b.id,
// crs, , crs_transform, dimensions, data_type
}));
const geeSchemas = collection.summaries['gee:schema'] || [];
const stacPropertyMapping = {};
for (const schema of geeSchemas) {
if (schema.stac_name) {
stacPropertyMapping[schema.name] = schema.stac_name;
}
}

const properties = {};
for(const key in img.properties) {
if (!omitProperties.includes(key)) {
let newKey;
if (!key.includes(":")) {
if (stacPropertyMapping[key]) {
newKey = stacPropertyMapping[key];
if (newKey.includes(":")) {
const extPrefix = newKey.split(":")[0];
if (STAC_EXTENSIONS[extPrefix]) {
extensions.add(STAC_EXTENSIONS[extPrefix]);
}
}
}
else if (!key.includes(":")) {
newKey = `gee:${key.toLowerCase()}`;
}
else {
Expand All @@ -268,18 +324,37 @@ export default class DataCatalog {
properties[newKey] = img.properties[key];
}
}

properties.datetime = Utils.toISODate(img.properties["system:time_start"]);
if (img.properties["system:time_end"]) {
properties.start_datetime = Utils.toISODate(img.properties["system:time_start"]);
properties.end_datetime = Utils.toISODate(img.properties["system:time_end"]);
}

extensions.add(STAC_EXTENSIONS.version);
properties.version = String(img.version);
properties["eo:bands"] = bands;

const hasBands = Array.isArray(img.bands) && img.bands.length > 0;
const collectionBands = collection.summaries['eo:bands'] || [];
// Is it a spectral band or SAR?
const isEO = hasBands && collectionBands.some(band => {
const keys = Object.keys(band);
console.log(keys);
return keys.includes("common_name") || keys.includes("center_wavelength") || keys.includes("full_width_half_max");
})
const bandMap = {};
for(const band of collectionBands) {
bandMap[band.name] = band;
}
if (hasBands && isEO) {
extensions.add(STAC_EXTENSIONS.eo);
properties["eo:bands"] = img.bands.map(band => Object.assign({name: band.id}, bandMap[band.id]));
}

const links = [
{
rel: "self",
href: API.getUrl(`/collections/${collection}/items/${id}`),
href: API.getUrl(`/collections/${collection.id}/items/${id}`),
type: "application/geo+json"
},
{
Expand All @@ -289,12 +364,12 @@ export default class DataCatalog {
},
{
rel: "parent",
href: API.getUrl(`/collections/${collection}`),
href: API.getUrl(`/collections/${collection.id}`),
type: "application/json"
},
{
rel: "collection",
href: API.getUrl(`/collections/${collection}`),
href: API.getUrl(`/collections/${collection.id}`),
type: "application/json"
}
];
Expand All @@ -307,7 +382,75 @@ export default class DataCatalog {
}
};

if (this.serverContext.stacAssetDownload) {
if (hasBands) {
for (const imgBand of img.bands) {
// Base asset for the band
const asset = {
href: API.getUrl(`/assets/${img.id}?band=${imgBand.id}`),
type: "image/tiff; application=geotiff",
roles: ["data"]
};

// Projection
if (imgBand.crs_transform) {
extensions.add(STAC_EXTENSIONS.proj);
asset["proj:transform"] = imgBand.crs_transform;
}
if (imgBand.dimensions) {
extensions.add(STAC_EXTENSIONS.proj);
const largerIndex = imgBand.dimensions[0] > imgBand.dimensions[1] ? 0 : 1;
const ratio = imgBand.dimensions[largerIndex] / this.serverContext.stacAssetDownloadSize;
const x = Math.round(imgBand.dimensions[0] / ratio);
const y = Math.round(imgBand.dimensions[1] / ratio);
asset["proj:shape"] = [y, x];
}
if (imgBand.crs) {
if (imgBand.crs.startsWith("EPSG:")) {
extensions.add(STAC_EXTENSIONS.proj);
asset["proj:epsg"] = parseInt(imgBand.crs.substring(5), 10);
}
else {
asset["gee:crs"] = imgBand.crs;
}
}

const baseBand = bandMap[imgBand.id] || {};
// Move gsd to asset
if (baseBand.gsd) {
asset.gsd = baseBand.gsd;
}

// EO bands
if (baseBand && isEO) {
asset["eo:bands"] = [
Utils.pickFromObject(baseBand, ["name", "description", "common_name", "center_wavelength", "full_width_half_max"])
];
}

// Raster bands
let rasterBand = baseBand;
if (isEO) {
rasterBand = {
name: rasterBand.name,
scale: rasterBand["gee:scale"],
offset: rasterBand["gee:offset"]
};
}
const dataType = this.getDataTypeFromPixelType(imgBand.data_type);
if (dataType) {
rasterBand.data_type = dataType;
}
// It should contain more than just a name property
if (Utils.size(rasterBand) > 1) {
extensions.add(STAC_EXTENSIONS.raster);
asset["raster:bands"] = [rasterBand];
}

// Add asset
assets[imgBand.id] = asset;
}
}
else {
assets.data = {
href: API.getUrl(`/assets/${img.id}`),
type: "image/tiff; application=geotiff",
Expand All @@ -317,23 +460,37 @@ export default class DataCatalog {

const stac = {
stac_version: "1.0.0",
stac_extensions: [
"https://stac-extensions.github.io/eo/v1.1.0/schema.json",
"https://stac-extensions.github.io/version/v1.0.0/schema.json",
],
stac_extensions: Array.from(extensions),
type: "Feature",
id,
bbox,
geometry,
properties,
collection,
collection: collection.id,
links,
assets
};

return stac;
}

getDataTypeFromPixelType(geeDataType) {
if (Utils.isObject(geeDataType) && geeDataType.type === "PixelType") {
const types = geeDataTypeMapping[geeDataType.precision];
if (Array.isArray(types)) {
for(const type of types) {
if (geeDataType.min === type.min && geeDataType.max === type.max) {
return type.name;
}
}
}
else if (typeof types === 'string') {
return types;
}
}
return null;
}

fixCollectionOnce(c) {
// Fix invalid headers in markdown
if (typeof c.description === 'string') {
Expand Down Expand Up @@ -376,7 +533,7 @@ export default class DataCatalog {
}
}

c.stac_extensions = [STAC_DATACUBE_EXTENSION];
c.stac_extensions = [STAC_EXTENSIONS.cube];

if (!c.extent || !c.extent.spatial || !c.extent.spatial.bbox) {
console.log("Invalid spatial extent for " + c.id);
Expand Down
2 changes: 1 addition & 1 deletion src/utils/config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ export default class Config {
this.diskUsagePath = null;

this.defaultLogLevel = "info";
this.stacAssetDownload = false;
this.stacAssetDownloadSize = 2000; // > 0 and <= 2000

const config = Utils.require('../../config.json');
for(const c in config) {
Expand Down

0 comments on commit 158b6f4

Please sign in to comment.