diff --git a/soil_id/db.py b/soil_id/db.py index ea95c62..4ff0c00 100644 --- a/soil_id/db.py +++ b/soil_id/db.py @@ -31,115 +31,6 @@ def get_datastore_connection(): sys.exit(str(err)) -# us, global -def save_model_output( - plot_id, model_version, result_blob, soilIDRank_output_pd, mucompdata_cond_prob -): - """ - Save the output of the model to the 'landpks_soil_model' table. - """ - try: - conn = get_datastore_connection() - cur = conn.cursor() - - sql = """ - INSERT INTO landpks_soil_model - (plot_id, model_version, result_blob, soilIDRank_output_pd, mucompdata_cond_prob) - VALUES (%s, %s, %s, %s, %s) - """ - cur.execute( - sql, - ( - plot_id, - model_version, - result_blob, - soilIDRank_output_pd, - mucompdata_cond_prob, - ), - ) - conn.commit() - - except Exception as err: - logging.error(err) - conn.rollback() - return None - finally: - conn.close() - - -# us, global -def save_rank_output(record_id, model_version, rank_blob): - """ - Update the rank of the soil model in the 'landpks_soil_model' table. - """ - try: - conn = get_datastore_connection() - cur = conn.cursor() - - sql = """UPDATE landpks_soil_model - SET soilrank = %s - WHERE ID = %s AND model_version = %s""" - cur.execute(sql, (rank_blob, record_id, model_version)) - conn.commit() - - except Exception as err: - logging.error(err) - conn.rollback() - return None - finally: - conn.close() - - -# us, global -def load_model_output(plot_id): - """ - Load model output based on plot ID and model version. - """ - try: - conn = get_datastore_connection() - cur = conn.cursor() - model_version = 2 - sql = """SELECT ID, result_blob, soilIDRank_output_pd, mucompdata_cond_prob - FROM landpks_soil_model - WHERE plot_id = %s AND model_version = %s - ORDER BY ID DESC LIMIT 1""" - cur.execute(sql, plot_id, model_version) - results = cur.fetchall() - for row in results: - model_run = [row[0], row[1], row[2], row[3]] - return model_run - except Exception as err: - logging.error(err) - return None - finally: - conn.close() - - -# global only -def save_soilgrids_output(plot_id, model_version, soilgrids_blob): - """ - Save the output of the soil grids to the 'landpks_soilgrids_model' table. - """ - try: - conn = get_datastore_connection() - cur = conn.cursor() - - sql = """ - INSERT INTO landpks_soilgrids_model - (plot_id, model_version, soilgrids_blob) - VALUES (%s, %s, %s) - """ - cur.execute(sql, (plot_id, model_version, soilgrids_blob)) - conn.commit() - - except Exception as err: - logging.error(err) - conn.rollback() - return None - finally: - conn.close() - - # global (via extract_WISE_data) def get_WISE30sec_data(MUGLB_NEW_Select): """ diff --git a/soil_id/global_soil.py b/soil_id/global_soil.py index bbf45df..9645d81 100644 --- a/soil_id/global_soil.py +++ b/soil_id/global_soil.py @@ -1,8 +1,6 @@ # Standard libraries import collections import csv -import io -import json import re # local libraries @@ -11,14 +9,7 @@ # Third-party libraries import numpy as np import pandas as pd -from db import ( - get_WRB_descriptions, - getSG_descriptions, - load_model_output, - save_model_output, - save_rank_output, - save_soilgrids_output, -) +from db import get_WRB_descriptions, getSG_descriptions from scipy.stats import norm from services import get_soilgrids_classification_data, get_soilgrids_property_data from utils import ( @@ -57,7 +48,7 @@ ################################################################################################## # getSoilLocationBasedGlobal # ################################################################################################## -def getSoilLocationBasedGlobal(lon, lat, plot_id): +def getSoilLocationBasedGlobal(lon, lat): # Extract HWSD-WISE Data # Note: Need to convert HWSD shp to gpkg file wise_data = extract_WISE_data( @@ -363,7 +354,6 @@ def aggregated_data_layer(data, column_name): compName = mucompdata_cond_prob["compname_grp"].apply(lambda x: x.capitalize()).tolist() score = mucompdata_cond_prob["distance_score_norm"].round(3).tolist() rank_loc = mucompdata_cond_prob["Rank_Loc"].tolist() - model_version = 3 # Step 3: Construct ID list directly from the sorted and cleaned DataFrame ID = [] @@ -447,34 +437,8 @@ def aggregated_data_layer(data, column_name): ] # Save data - if plot_id is None: - soilIDRank_output_pd.to_csv(config.SOIL_ID_RANK_PATH, index=None, header=True) - mucompdata_cond_prob.to_csv(config.SOIL_ID_PROB_PATH, index=None, header=True) - else: - save_model_output( - plot_id, - model_version, - json.dumps( - { - "metadata": { - "location": "global", - "model": "v2", - "unit_measure": { - "distance": "m", - "depth": "cm", - "cec": "cmol(c)/kg", - "clay": "%", - "rock_fragments": "cm3/100cm3", - "sand": "%", - "ec": "ds/m", - }, - }, - "soilList": output_SoilList, - } - ), - soilIDRank_output_pd.to_csv(index=None, header=True), - mucompdata_cond_prob.to_csv(index=None, header=True), - ) + soilIDRank_output_pd.to_csv(config.SOIL_ID_RANK_PATH, index=None, header=True) + mucompdata_cond_prob.to_csv(config.SOIL_ID_PROB_PATH, index=None, header=True) # Return the JSON output return { @@ -498,9 +462,7 @@ def aggregated_data_layer(data, column_name): ############################################################################################## # rankPredictionGlobal # ############################################################################################## -def rankPredictionGlobal( - lon, lat, soilHorizon, horizonDepth, rfvDepth, lab_Color, bedrock, cracks, plot_id=None -): +def rankPredictionGlobal(lon, lat, soilHorizon, horizonDepth, rfvDepth, lab_Color, bedrock, cracks): # ------------------------------------------------------------------------------------------------ # ------ Load in user data --------# @@ -658,25 +620,9 @@ def rankPredictionGlobal( # -------------------------------------------------------------------------------------------------------------------------------------- # Load in component data from soilIDList - # Initialize - record_id = None - - # If no plot_id is provided, load data from file - if plot_id is None: - soilIDRank_output_pd = pd.read_csv(config.SOIL_ID_RANK_PATH) - mucompdata_pd = pd.read_csv(config.SOIL_ID_PROB_PATH) - # If plot_id is provided, load data from the database - else: - modelRun = load_model_output(plot_id) - - # Check if modelRun data was successfully fetched - if modelRun: - record_id = modelRun[0] - soilIDRank_output_pd = pd.read_csv(io.StringIO(modelRun[2])) - mucompdata_pd = pd.read_csv(io.StringIO(modelRun[3])) - else: - return "Cannot find a plot with this ID" + soilIDRank_output_pd = pd.read_csv(config.SOIL_ID_RANK_PATH) + mucompdata_pd = pd.read_csv(config.SOIL_ID_PROB_PATH) # Create soil depth DataFrame and subset component depths based on max user depth # if no bedrock specified @@ -1207,7 +1153,6 @@ def rankPredictionGlobal( rank_list.append(rank_entry) # Setting up the return data structure - model_version = 3 metadata = { "location": "global", "model": "v3", @@ -1216,17 +1161,13 @@ def rankPredictionGlobal( result = {"metadata": metadata, "soilRank": rank_list} - # Save data if record_id is provided - if record_id: - save_rank_output(record_id, model_version, json.dumps(result)) - return result ################################################################################################## # getSoilGridsGlobal # ################################################################################################## -def getSoilGridsGlobal(lon, lat, plot_id=None): +def getSoilGridsGlobal(lon, lat): # Call soildgrids API sg_out = get_soilgrids_property_data(lon, lat) @@ -1262,8 +1203,6 @@ def getSoilGridsGlobal(lon, lat, plot_id=None): # Check if all values in the specified columns are NaN if sg_data_w[["sand", "clay", "cfvo"]].isnull().all().all(): - if plot_id is not None: - save_soilgrids_output(plot_id, 1, json.dumps({"status": "unavailable"})) return {"status": "unavailable"} else: # Apply the factor to the specific columns @@ -1273,7 +1212,7 @@ def getSoilGridsGlobal(lon, lat, plot_id=None): sg_data_w["silt"] = sg_data_w.apply(silt_calc, axis=1) sg_data_w["texture"] = sg_data_w.apply(getTexture, axis=1) - sg_tax = get_soilgrids_classification_data(lon, lat, plot_id) + sg_tax = get_soilgrids_classification_data(lon, lat) # If data was successfully fetched, process it if sg_tax: @@ -1323,9 +1262,6 @@ def getSoilGridsGlobal(lon, lat, plot_id=None): texture_pd["silt"] = texture_pd.apply(silt_calc, axis=1) texture_pd_lpks = texture_pd.apply(getTexture, axis=1).replace([None], "") - # SoilGrids API call version 1 - model_version = 1 - # Define keys and corresponding values for the 'components' dictionary component_keys = [ "compname", @@ -1392,11 +1328,5 @@ def getSoilGridsGlobal(lon, lat, plot_id=None): }, } - # If a plot_id is provided, save the SoilGrids output - if plot_id is not None: - save_soilgrids_output( - plot_id, model_version, json.dumps({"metadata": metadata, "soilGrids": SoilGrids}) - ) - # Return the final result return {"metadata": metadata, "soilGrids": SoilGrids} diff --git a/soil_id/services.py b/soil_id/services.py index 91df856..f3c2c8d 100644 --- a/soil_id/services.py +++ b/soil_id/services.py @@ -1,4 +1,3 @@ -import json import logging import re @@ -7,8 +6,6 @@ import requests from pandas import json_normalize -from .db import save_soilgrids_output - def get_elev_data(lon, lat): """ @@ -157,9 +154,6 @@ def get_soilgrids_property_data(lon, lat, plot_id): logging.error("Soilgrids properties: timed out") except requests.RequestException as err: logging.error(f"Soilgrids properties: error: {err}") - if plot_id is not None: - # Assuming the function `save_soilgrids_output` exists elsewhere in the code - save_soilgrids_output(plot_id, 1, json.dumps({"status": "unavailable"})) return result if result is not None else {"status": "unavailable"} @@ -182,9 +176,6 @@ def get_soilgrids_classification_data(lon, lat, plot_id): logging.error("Soilgrids classification: timed out") except requests.RequestException as err: logging.error(f"Soilgrids classification: error: {err}") - if plot_id is not None: - # Assuming the function `save_soilgrids_output` exists elsewhere in the code - save_soilgrids_output(plot_id, 1, json.dumps({"status": "unavailable"})) return result