Skip to content

Commit

Permalink
manifest updating properly
Browse files Browse the repository at this point in the history
  • Loading branch information
GondekNP committed Jan 6, 2024
1 parent 7dbedfd commit bc5fcfa
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 31 deletions.
9 changes: 6 additions & 3 deletions app.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
app.include_router(cog.router, prefix='/cog', tags=["Cloud Optimized GeoTIFF"])
add_exception_handlers(app, DEFAULT_STATUS_CODES)

logging_client = logging.Client()
logging_client = logging.Client(project='dse-nps')
log_name = "burn-backend"
logger = logging_client.logger(log_name)

Expand Down Expand Up @@ -114,8 +114,11 @@ def analyze_burn(body: AnaylzeBurnPOSTBody, sftp_client: SFTPClient = Depends(ge

# save the cog to the FTP server
sftp_client.connect()
sftp_client.upload_cogs(
metrics_stack=geo_client.metrics_stack, fire_event_name=fire_event_name
sftp_client.upload_fire_event(
metrics_stack=geo_client.metrics_stack,
fire_event_name=fire_event_name,
prefire_date_range=date_ranges["prefire"],
postfire_date_range=date_ranges["postfire"]
)
sftp_client.disconnect()
logger.log_text(f"Cogs uploaded for {fire_event_name}")
Expand Down
88 changes: 60 additions & 28 deletions src/util/sftp.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def __init__(self, hostname, username, private_key, port=22):
self.available_cogs = None

# Set up logging
logging_client = cloud_logging.Client()
logging_client = cloud_logging.Client(project='dse-nps')
log_name = "burn-backend"
self.logger = logging_client.logger(log_name)

Expand Down Expand Up @@ -78,9 +78,6 @@ def download(self, remote_path, target_local_path):
"""

try:
print(
f"downloading from {self.hostname} as {self.username} [(remote path : {remote_path});(local path: {target_local_path})]"
)

# Create the target directory if it does not exist
path, _ = os.path.split(target_local_path)
Expand All @@ -92,7 +89,6 @@ def download(self, remote_path, target_local_path):

# Download from remote sftp server to local
self.connection.get(remote_path, target_local_path)
print("download completed")

except Exception as err:
raise Exception(err)
Expand All @@ -114,29 +110,6 @@ def upload(self, source_local_path, remote_path):
except Exception as err:
raise Exception(err)

def upload_cogs(self, metrics_stack, fire_event_name):
# Save our stack to a COG, in a tempfile
with tempfile.TemporaryDirectory() as tmpdir:

self.download('manifest.json', 'tmp_manifest.json')
self.logger.log_text(f"Downloaded manifest.json: {manifest}")
manifest = json.load(open('tmp_manifest.json', 'r'))

if fire_event_name in manifest:
self.logger.log_text(f"Fire event {fire_event_name} already exists in manifest. Overwriting.")
del manifest[fire_event_name]


for band_name in metrics_stack.burn_metric.to_index():
local_cog_path = os.path.join(tmpdir, f"{band_name}.tif")
band_cog = metrics_stack.sel(burn_metric = band_name).rio
band_cog.to_raster(local_cog_path)
# Upload the metrics to our SFTP server
self.upload(
source_local_path=local_cog_path,
remote_path=f"{fire_event_name}/{band_name}.tif",
)

def get_available_cogs(self):
"""Lists all available COGs on the SFTP server"""
available_cogs = {}
Expand All @@ -151,3 +124,62 @@ def update_available_cogs(self):
self.connect()
self.available_cogs = self.get_available_cogs()
self.disconnect()

def upload_cogs(self, metrics_stack, fire_event_name, prefire_date_range, postfire_date_range):

with tempfile.TemporaryDirectory() as tmpdir:

for band_name in metrics_stack.burn_metric.to_index():
local_cog_path = os.path.join(tmpdir, f"{band_name}.tif")
band_cog = metrics_stack.sel(burn_metric = band_name).rio
band_cog.to_raster(local_cog_path, driver="GTiff")
self.upload(
source_local_path=local_cog_path,
remote_path=f"{fire_event_name}/{band_name}.tif",
)

def update_manifest(self, fire_event_name, bounds, prefire_date_range, postfire_date_range):
with tempfile.TemporaryDirectory() as tmpdir:
self.download('manifest.json', '/tmp_manifest.json')
self.logger.log_text(f"Downloaded manifest.json")
manifest = json.load(open('/tmp_manifest.json', 'r'))

if fire_event_name in manifest:
self.logger.log_text(f"Fire event {fire_event_name} already exists in manifest. Overwriting.")
del manifest[fire_event_name]

manifest[fire_event_name] = {
'bounds': bounds,
'prefire_date_range': prefire_date_range,
'postfire_date_range': postfire_date_range,
'last_updated': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
}

# Upload the manifest to our SFTP server
tmp_manifest_path = os.path.join(tmpdir, 'manifest_updated.json')
with open(tmp_manifest_path, 'w') as f:
json.dump(manifest, f)
self.upload(
source_local_path=tmp_manifest_path,
remote_path='manifest.json'
)
self.logger.log_text(f"Uploaded/updated manifest.json")

def upload_fire_event(self, metrics_stack, fire_event_name, prefire_date_range, postfire_date_range):
self.logger.log_text(f"Uploading fire event {fire_event_name}")

self.upload_cogs(
metrics_stack=metrics_stack,
fire_event_name=fire_event_name,
prefire_date_range=prefire_date_range,
postfire_date_range=postfire_date_range
)

bounds = [round(pos, 4) for pos in metrics_stack.rio.bounds()]

self.update_manifest(
fire_event_name=fire_event_name,
bounds=bounds,
prefire_date_range=prefire_date_range,
postfire_date_range=postfire_date_range
)

0 comments on commit bc5fcfa

Please sign in to comment.