diff --git a/.github/dependabot.yml b/.github/dependabot.yml index df1c2e0..feee612 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,7 +4,7 @@ updates: directory: "/" schedule: interval: daily - - package-ecosystem: 'github-actions' - directory: '/' + - package-ecosystem: "github-actions" + directory: "/" schedule: - interval: 'daily' + interval: "daily" diff --git a/carbonplan_trace/fire/grid_viirs_workflow.py b/carbonplan_trace/fire/grid_viirs_workflow.py index b300b93..806087c 100755 --- a/carbonplan_trace/fire/grid_viirs_workflow.py +++ b/carbonplan_trace/fire/grid_viirs_workflow.py @@ -35,7 +35,6 @@ def calc_buffer_distance(lats, buffer_m=375): def rasterize_geom(geoms, transform, shape): - r = rasterize( [(geom, 1) for geom in geoms], out_shape=shape, @@ -85,7 +84,6 @@ def open_target_grid(tile_id): def process_one_year(year): - print(year) gdf = geopandas.read_parquet( f's3://carbonplan-climatetrace/inputs/processed/viirs/{year}.parquet' diff --git a/carbonplan_trace/metadata.py b/carbonplan_trace/metadata.py index 936b30f..1dde5c3 100644 --- a/carbonplan_trace/metadata.py +++ b/carbonplan_trace/metadata.py @@ -8,9 +8,8 @@ def get_cf_global_attrs(**attrs): - if 'history' not in attrs: - attrs['history'] = 'Created: {}'.format(time.ctime(time.time())) + attrs['history'] = f'Created: {time.ctime(time.time())}' if 'insitution' not in attrs: attrs['institution'] = 'CarbonPlan' diff --git a/carbonplan_trace/v0/data/catalog.yaml b/carbonplan_trace/v0/data/catalog.yaml index 5c566c9..0b41c70 100644 --- a/carbonplan_trace/v0/data/catalog.yaml +++ b/carbonplan_trace/v0/data/catalog.yaml @@ -3,11 +3,10 @@ plugins: - module: intake_xarray sources: - hansen_change: description: Global 30m treecover and loss/gain metadata: - url: 'https://storage.cloud.google.com/earthenginepartners-hansen/GFC-2020-v1.8/download.html' + url: "https://storage.cloud.google.com/earthenginepartners-hansen/GFC-2020-v1.8/download.html" tags: - forest parameters: @@ -25,13 +24,13 @@ sources: default: 060W driver: rasterio args: - urlpath: 'https://storage.googleapis.com/earthenginepartners-hansen/GFC-2020-v1.8/Hansen_GFC-2020-v1.8_{{ variable }}_{{ lat }}_{{ lon }}.tif' - chunks: {'y': 4000, 'x': 4000} + urlpath: "https://storage.googleapis.com/earthenginepartners-hansen/GFC-2020-v1.8/Hansen_GFC-2020-v1.8_{{ variable }}_{{ lat }}_{{ lon }}.tif" + chunks: { "y": 4000, "x": 4000 } gfw_biomass: description: Global 30m above ground biomass metadata: - url: 'http://data.globalforestwatch.org/datasets/8f93a6f94a414f9588ce4657a39c59ff_1' + url: "http://data.globalforestwatch.org/datasets/8f93a6f94a414f9588ce4657a39c59ff_1" tags: - biomass parameters: @@ -45,125 +44,125 @@ sources: default: 060W driver: rasterio args: - urlpath: 'http://gfw2-data.s3.amazonaws.com/climate/WHRC_biomass/WHRC_V4/Processed/{{ lat }}_{{lon}}_t_aboveground_biomass_ha_2000.tif' - chunks: {'y': 4000, 'x': 4000} + urlpath: "http://gfw2-data.s3.amazonaws.com/climate/WHRC_biomass/WHRC_V4/Processed/{{ lat }}_{{lon}}_t_aboveground_biomass_ha_2000.tif" + chunks: { "y": 4000, "x": 4000 } gfw_emissions: - description: Global 30m emissions from Harris et al. 2021 - metadata: - url: 'https://data.globalforestwatch.org/datasets/forest-greenhouse-gas-emissions-1' - tags: - - biomass - parameters: - lat: - description: latitude of northwest corner of tile - type: str - default: 00N - lon: - description: longitude of northwest corner of tile - type: str - default: 060W - unit: - description: emission units - type: str - default: Mg_CO2e_ha - driver: rasterio - args: - urlpath: 'https://data-api.globalforestwatch.org/dataset/gfw_forest_carbon_gross_emissions/v20210331/download/geotiff?grid=10/40000&tile_id={{ lat }}_{{lon}}&pixel_meaning={{ unit }}' - chunks: {'y': 4000, 'x': 4000} + description: Global 30m emissions from Harris et al. 2021 + metadata: + url: "https://data.globalforestwatch.org/datasets/forest-greenhouse-gas-emissions-1" + tags: + - biomass + parameters: + lat: + description: latitude of northwest corner of tile + type: str + default: 00N + lon: + description: longitude of northwest corner of tile + type: str + default: 060W + unit: + description: emission units + type: str + default: Mg_CO2e_ha + driver: rasterio + args: + urlpath: "https://data-api.globalforestwatch.org/dataset/gfw_forest_carbon_gross_emissions/v20210331/download/geotiff?grid=10/40000&tile_id={{ lat }}_{{lon}}&pixel_meaning={{ unit }}" + chunks: { "y": 4000, "x": 4000 } emissions_30m_partitioned: - description: Global 30m emissions from CarbonPlan partitioned into fire and non-fire related components. See carbonplan.org/blog/climate-trace-release for more information. - metadata: - url: '' - tags: - - emissions_partitioned - parameters: - lat: - description: latitude of northwest corner of tile - type: str - default: 00N - lon: - description: longitude of northwest corner of tile - type: str - default: 060W - unit: - description: emission units - type: str - default: Mg_CO2e_ha - driver: zarr - args: - urlpath: 'https://carbonplan-climatetrace.s3.us-west-2.amazonaws.com/v0.4/tiles/30m/{{ lat }}_{{ lon }}_split.zarr' - consolidated: True + description: Global 30m emissions from CarbonPlan partitioned into fire and non-fire related components. See carbonplan.org/blog/climate-trace-release for more information. + metadata: + url: "" + tags: + - emissions_partitioned + parameters: + lat: + description: latitude of northwest corner of tile + type: str + default: 00N + lon: + description: longitude of northwest corner of tile + type: str + default: 060W + unit: + description: emission units + type: str + default: Mg_CO2e_ha + driver: zarr + args: + urlpath: "https://carbonplan-climatetrace.s3.us-west-2.amazonaws.com/v0.4/tiles/30m/{{ lat }}_{{ lon }}_split.zarr" + consolidated: True emissions_30m: - description: Global 30 m emissions from CarbonPlan - metadata: - url: '' - tags: - - emissions - parameters: - lat: - description: latitude of northwest corner of tile - type: str - default: 00N - lon: - description: longitude of northwest corner of tile - type: str - default: 060W - unit: - description: emission units - type: str - default: Mg_CO2e_ha - driver: zarr - args: - urlpath: 'https://carbonplan-climatetrace.s3.us-west-2.amazonaws.com/v0.4/tiles/30m/{{ lat }}_{{ lon }}_tot.zarr' - consolidated: True + description: Global 30 m emissions from CarbonPlan + metadata: + url: "" + tags: + - emissions + parameters: + lat: + description: latitude of northwest corner of tile + type: str + default: 00N + lon: + description: longitude of northwest corner of tile + type: str + default: 060W + unit: + description: emission units + type: str + default: Mg_CO2e_ha + driver: zarr + args: + urlpath: "https://carbonplan-climatetrace.s3.us-west-2.amazonaws.com/v0.4/tiles/30m/{{ lat }}_{{ lon }}_tot.zarr" + consolidated: True emissions_3km: - description: Global 3 km emissions from CarbonPlan - metadata: - url: '' - tags: - - emissions - parameters: - unit: - description: emission units - type: str - default: Mg_CO2e - driver: zarr - args: - urlpath: 'https://carbonplan-climatetrace.s3.us-west-2.amazonaws.com/v0.4/global/3000m/raster_tot.zarr' - consolidated: True + description: Global 3 km emissions from CarbonPlan + metadata: + url: "" + tags: + - emissions + parameters: + unit: + description: emission units + type: str + default: Mg_CO2e + driver: zarr + args: + urlpath: "https://carbonplan-climatetrace.s3.us-west-2.amazonaws.com/v0.4/global/3000m/raster_tot.zarr" + consolidated: True emissions_3km_partitioned: - description: Global 3 km emissions from CarbonPlan partitioned into fire and non-fire related components - metadata: - url: '' - tags: - - emissions - parameters: - unit: - description: emission units - type: str - default: Mg_CO2e - driver: zarr - args: - urlpath: 'https://carbonplan-climatetrace.s3.us-west-2.amazonaws.com/v0.4/global/3000m/raster_split.zarr' - consolidated: True + description: Global 3 km emissions from CarbonPlan partitioned into fire and non-fire related components + metadata: + url: "" + tags: + - emissions + parameters: + unit: + description: emission units + type: str + default: Mg_CO2e + driver: zarr + args: + urlpath: "https://carbonplan-climatetrace.s3.us-west-2.amazonaws.com/v0.4/global/3000m/raster_split.zarr" + consolidated: True emissions_one_degree: - description: Global 1-degree resolution emissions from CarbonPlan - metadata: - url: '' - tags: - - emissions - parameters: - unit: - description: emission units - type: str - default: Mg_CO2e - driver: zarr - args: - urlpath: 'https://carbonplan-climatetrace.s3.us-west-2.amazonaws.com/v0.4/blog/total_emissions.zarr' - consolidated: True + description: Global 1-degree resolution emissions from CarbonPlan + metadata: + url: "" + tags: + - emissions + parameters: + unit: + description: emission units + type: str + default: Mg_CO2e + driver: zarr + args: + urlpath: "https://carbonplan-climatetrace.s3.us-west-2.amazonaws.com/v0.4/blog/total_emissions.zarr" + consolidated: True diff --git a/carbonplan_trace/v0/workflow.py b/carbonplan_trace/v0/workflow.py index 6a8615a..ec80658 100755 --- a/carbonplan_trace/v0/workflow.py +++ b/carbonplan_trace/v0/workflow.py @@ -78,7 +78,6 @@ def process_one_tile(tile_id): # calc emissions if not (skip_existing and zarr_is_complete(tot_mapper) and zarr_is_complete(split_mapper)): - lat, lon = tile_id.split('_') fire_da = open_fire_mask(tile_id).fillna(0) change_ds = open_hansen_change_tile(lat, lon) @@ -112,7 +111,6 @@ def process_one_tile(tile_id): (tot_mapper, coarse_tot_mapper, tot_encoding), (split_mapper, coarse_split_mapper, split_encoding), ]: - if not (skip_existing and zarr_is_complete(out_mapper)): ds = xr.open_zarr(in_mapper, consolidated=True) @@ -134,7 +132,6 @@ def combine_all_tiles(encoding_kinds=None): encoding_kinds = [('tot', tot_encoding), ('split', split_encoding)] print('combining all tiles') for kind, encoding in encoding_kinds: - mapper = fsspec.get_mapper(coarse_full_template.format(kind=kind)) if not (skip_existing and zarr_is_complete(mapper)): @@ -169,13 +166,11 @@ def rollup_shapes(): ('tot', ['emissions']), ('split', ['emissions_from_clearing', 'emissions_from_fire']), ]: - ds = xr.open_zarr(coarse_full_template.format(kind=kind), consolidated=True) mask = regionmask.mask_geopandas(shapes_df, ds['lon'], ds['lat'], numbers='numbers') for var in var_names: - # this will trigger dask compute df = ds[var].groupby(mask).sum().to_pandas() diff --git a/carbonplan_trace/v1/emissions_workflow.py b/carbonplan_trace/v1/emissions_workflow.py index 7f50ed5..79804cf 100755 --- a/carbonplan_trace/v1/emissions_workflow.py +++ b/carbonplan_trace/v1/emissions_workflow.py @@ -207,7 +207,6 @@ def coarsen_tile(tile_id): for in_mapper, out_mapper, encoding in [ (split_mapper, coarse_split_mapper, split_encoding), ]: - if not (skip_existing and zarr_is_complete(out_mapper)): ds = utils.open_result_tile( tile_id, variable='emissions', version='v1.2', resolution='30m', apply_masks=True @@ -233,7 +232,6 @@ def combine_all_tiles(encoding_kinds=None): print('combining all tiles') for kind, encoding in encoding_kinds: - mapper = fsspec.get_mapper(coarse_full_template.format(kind=kind)) if not (skip_existing and zarr_is_complete(mapper)): @@ -270,7 +268,6 @@ def rollup_shapes(): var_names = ['emissions_from_clearing', 'emissions_from_fire', 'sinks', 'net'] for var in var_names: - # this will trigger dask compute df = ds[var].groupby(mask).sum().to_pandas() diff --git a/carbonplan_trace/v1/inference.py b/carbonplan_trace/v1/inference.py index 64a1b94..7a57e08 100644 --- a/carbonplan_trace/v1/inference.py +++ b/carbonplan_trace/v1/inference.py @@ -39,7 +39,7 @@ def write_crs_dataset(ds, zone=None, overwrite=False): one already exists!) so be sure you want to do that! ''' if zone is None: - zone = '{}{}'.format(ds.utm_zone_number, ds.utm_zone_letter) + zone = f'{ds.utm_zone_number}{ds.utm_zone_letter}' crs = CRS.from_dict({'proj': 'utm', 'zone': zone}) ds = ds.rio.set_crs(crs) return ds diff --git a/carbonplan_trace/v1/landsat_preprocess.py b/carbonplan_trace/v1/landsat_preprocess.py index ae1607b..3b38892 100644 --- a/carbonplan_trace/v1/landsat_preprocess.py +++ b/carbonplan_trace/v1/landsat_preprocess.py @@ -88,11 +88,10 @@ def grab_ds(item, bands_of_interest, cog_mask, utm_zone, utm_letter): ''' if type(item) == str: - url_list = [item + '_{}.TIF'.format(band) for band in bands_of_interest] + url_list = [item + f'_{band}.TIF' for band in bands_of_interest] else: url_list = [ - fix_link(item._stac_obj.assets['{}.TIF'.format(band)]['href']) - for band in bands_of_interest + fix_link(item._stac_obj.assets[f'{band}.TIF']['href']) for band in bands_of_interest ] da_list = [] for url in url_list: @@ -265,7 +264,6 @@ def grab_scene_coord_info(metadata): def get_scene_utm_info(url, json_client): - ''' Get the USGS-provided UTM zone and letter for the specific landsat scene @@ -330,7 +328,6 @@ def calc_NDVI(ds): def calc_NDII(ds): - ''' Calculate NDII (Hardisky et al, 1984) based upon bands 4 and 5. *Note* only valid for landsat 5 and 7 right now. @@ -379,7 +376,7 @@ def make_datestamps(months, year): include landsat scenes from jan/feb 2004. ''' years = [year + 1 if int(month) < 3 else year for month in months] - return ['{}{}'.format(year, month) for (year, month) in zip(years, months)] + return [f'{year}{month}' for (year, month) in zip(years, months)] def scene_seasonal_average( @@ -425,7 +422,7 @@ def scene_seasonal_average( valid_files.append(scene_store) for file in valid_files: scene_id = file[-40:] - url = 's3://{}/{}'.format(file, scene_id) + url = f's3://{file}/{scene_id}' # try: utm_zone, utm_letter = get_scene_utm_info(url, test_client) cloud_mask_url = url + '_SR_CLOUD_QA.TIF' diff --git a/carbonplan_trace/v1/load.py b/carbonplan_trace/v1/load.py index 050caca..929e240 100644 --- a/carbonplan_trace/v1/load.py +++ b/carbonplan_trace/v1/load.py @@ -211,9 +211,7 @@ def biomass(tiles, year): complete_df = None for tile in tiles: if tile not in no_biomass_data_tiles: - file_mapper = fs.get_mapper( - 's3://carbonplan-climatetrace/v1/biomass/{}.zarr'.format(tile) - ) + file_mapper = fs.get_mapper(f's3://carbonplan-climatetrace/v1/biomass/{tile}.zarr') ds = xr.open_zarr(file_mapper, consolidated=True) df = ds.stack(unique_index=("record_index", "shot_number")).to_dataframe() df = df[df['biomass'].notnull()] diff --git a/carbonplan_trace/v1/postprocess.py b/carbonplan_trace/v1/postprocess.py index dd383f1..e9a415a 100644 --- a/carbonplan_trace/v1/postprocess.py +++ b/carbonplan_trace/v1/postprocess.py @@ -413,7 +413,6 @@ def postprocess_subtile(parameters_dict): def test_to_zarr(parameters_dict): - lat_increment = parameters_dict['LAT_INCREMENT'] lon_increment = parameters_dict['LON_INCREMENT'] year0 = parameters_dict['YEAR_0'] @@ -438,7 +437,6 @@ def test_to_zarr(parameters_dict): data_mapper = fs.get_mapper(data_path) with rio.Env(aws_session): - da = xr.DataArray( np.ones((8000, 8000, 2)), coords={ diff --git a/carbonplan_trace/v1/training_prep.py b/carbonplan_trace/v1/training_prep.py index 16299fb..0232ce6 100644 --- a/carbonplan_trace/v1/training_prep.py +++ b/carbonplan_trace/v1/training_prep.py @@ -203,9 +203,8 @@ def add_parquet_urls(df): def find_pertinent_scenes_for_shots(lat_tag, lon_tag, scenes_in_tile_gdf): - file_mapper = fs.get_mapper( - "carbonplan-climatetrace/v2/data/intermediates/biomass/{}_{}.zarr".format(lat_tag, lon_tag) + f"carbonplan-climatetrace/v2/data/intermediates/biomass/{lat_tag}_{lon_tag}.zarr" ) biomass = xr.open_zarr(file_mapper, consolidated=True).load().drop("spatial_ref") @@ -235,7 +234,6 @@ def find_pertinent_scenes_for_shots(lat_tag, lon_tag, scenes_in_tile_gdf): def aggregate_parquet_files( lat_tag, lon_tag, all_parquet_files, write=True, access_key_id=None, secret_access_key=None ): - full_df = None for url in all_parquet_files: df = pd.read_parquet(url) @@ -245,7 +243,7 @@ def aggregate_parquet_files( full_df = pd.concat([full_df, df]) if write: tile_parquet_file_path = fs.get_mapper( - "carbonplan-climatetrace/v2/training/tiles/data_{}_{}.parquet".format(lat_tag, lon_tag) + f"carbonplan-climatetrace/v2/training/tiles/data_{lat_tag}_{lon_tag}.parquet" ) utils.write_parquet(df, tile_parquet_file_path, access_key_id, secret_access_key) else: diff --git a/data/data.json b/data/data.json index 134b1ed..d4d00be 100644 --- a/data/data.json +++ b/data/data.json @@ -446,31 +446,31 @@ "Bahamas": [ { "year": 2001, - "emissions": 7.840994270584921e-06 + "emissions": 7.840994270584921e-6 }, { "year": 2002, - "emissions": 2.678096706833859e-05 + "emissions": 2.678096706833859e-5 }, { "year": 2003, - "emissions": 3.588671841991143e-06 + "emissions": 3.588671841991143e-6 }, { "year": 2004, - "emissions": 2.223648978284003e-05 + "emissions": 2.223648978284003e-5 }, { "year": 2005, - "emissions": 2.126096562368698e-05 + "emissions": 2.126096562368698e-5 }, { "year": 2006, - "emissions": 4.460699115679086e-05 + "emissions": 4.460699115679086e-5 }, { "year": 2007, - "emissions": 3.444218995420521e-05 + "emissions": 3.444218995420521e-5 }, { "year": 2008, @@ -478,35 +478,35 @@ }, { "year": 2009, - "emissions": 2.884173148261978e-05 + "emissions": 2.884173148261978e-5 }, { "year": 2010, - "emissions": 1.0801684506491563e-05 + "emissions": 1.0801684506491563e-5 }, { "year": 2011, - "emissions": 4.146125586837315e-05 + "emissions": 4.146125586837315e-5 }, { "year": 2012, - "emissions": 1.6816146920761377e-05 + "emissions": 1.6816146920761377e-5 }, { "year": 2013, - "emissions": 5.430238979265424e-05 + "emissions": 5.430238979265424e-5 }, { "year": 2014, - "emissions": 5.786899673843161e-05 + "emissions": 5.786899673843161e-5 }, { "year": 2015, - "emissions": 7.073263599589283e-05 + "emissions": 7.073263599589283e-5 }, { "year": 2016, - "emissions": 7.40113397210245e-05 + "emissions": 7.40113397210245e-5 }, { "year": 2017, @@ -514,7 +514,7 @@ }, { "year": 2018, - "emissions": 7.723572634661022e-05 + "emissions": 7.723572634661022e-5 } ], "Falkland Is.": [ diff --git a/notebooks/analysis/above_data.ipynb b/notebooks/analysis/above_data.ipynb index d183d7a..6a6ce10 100644 --- a/notebooks/analysis/above_data.ipynb +++ b/notebooks/analysis/above_data.ipynb @@ -18,23 +18,8 @@ "metadata": {}, "outputs": [], "source": [ - "import geopandas as gpd\n", - "import pandas as pd\n", "import numpy as np\n", "import xarray as xr\n", - "import matplotlib.pyplot as plt\n", - "\n", - "import os\n", - "import shutil\n", - "import regionmask\n", - "import rioxarray\n", - "\n", - "from itertools import product\n", - "from zarr.errors import GroupNotFoundError\n", - "\n", - "from shapely.geometry import Polygon\n", - "\n", - "import carbonplan_trace.v1.utils as utils\n", "from s3fs import S3FileSystem\n", "\n", "fs = S3FileSystem()" @@ -177,15 +162,10 @@ "metadata": {}, "outputs": [], "source": [ - "import xarray as xr\n", - "from numcodecs.zlib import Zlib\n", - "import pandas as pd\n", - "import numpy as np\n", "import dask\n", + "import numpy as np\n", + "import xarray as xr\n", "from s3fs import S3FileSystem\n", - "import geopandas\n", - "from carbonplan_trace.v1 import utils\n", - "import regionmask\n", "\n", "fs = S3FileSystem()\n", "\n", diff --git a/notebooks/analysis/biomass_rollup.ipynb b/notebooks/analysis/biomass_rollup.ipynb index f127fc1..27c51ac 100644 --- a/notebooks/analysis/biomass_rollup.ipynb +++ b/notebooks/analysis/biomass_rollup.ipynb @@ -10,29 +10,25 @@ "%load_ext autoreload\n", "%autoreload 2\n", "\n", + "from collections import defaultdict\n", + "\n", "import dask\n", + "import fsspec\n", + "import geopandas as gpd\n", + "import h5py\n", "import numpy as np\n", "import pandas as pd\n", + "import regionmask\n", "import xarray as xr\n", "\n", - "from carbonplan_trace.tiles import tiles\n", - "from carbonplan_trace.v1 import biomass_rollup\n", - "\n", - "import fsspec\n", - "import h5py\n", - "import regionmask\n", - "from rasterio.session import AWSSession\n", + "# from dask.distributed import Client\n", + "from carbonplan_data.utils import set_zarr_encoding\n", "from rasterio.warp import Resampling\n", "\n", - "from carbonplan_trace.v1 import utils\n", - "from collections import defaultdict\n", - "from carbonplan_trace.v1.landsat_preprocess import access_credentials\n", - "from carbonplan_trace.v0.core import compute_grid_area, coarsen_emissions\n", - "import rioxarray as rio\n", - "import geopandas as gpd\n", - "\n", - "# from dask.distributed import Client\n", - "from carbonplan_data.utils import set_zarr_encoding" + "from carbonplan_trace.tiles import tiles\n", + "from carbonplan_trace.v0.core import coarsen_emissions, compute_grid_area\n", + "from carbonplan_trace.v1 import biomass_rollup, utils\n", + "from carbonplan_trace.v1.landsat_preprocess import access_credentials" ] }, { @@ -220,7 +216,7 @@ "# we already have harris biomass summed by a factor of 100\n", "input_filename = \"s3://carbonplan-climatetrace/v0.4/global/3000m/raster_biomass.zarr\"\n", "# we want to get harris biomass averaged by a factor of 100\n", - "output_filename = f\"s3://carbonplan-climatetrace/validation/harris_3000m_agbd.zarr\"\n", + "output_filename = \"s3://carbonplan-climatetrace/validation/harris_3000m_agbd.zarr\"\n", "var = \"agb\"\n", "# we have total land area of each 3km grid in this file to divide the sum by\n", "land_area_filename = (\n", @@ -297,8 +293,7 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "import subprocess" + "import os" ] }, { @@ -309,7 +304,7 @@ "outputs": [], "source": [ "folder = \"/home/jovyan/globbiomass/\"\n", - "all_tiles = list(set([t.split(\"_\")[0] for t in os.listdir(folder)]))" + "all_tiles = list({t.split(\"_\")[0] for t in os.listdir(folder)})" ] }, { @@ -319,7 +314,7 @@ "metadata": {}, "outputs": [], "source": [ - "harris = xr.open_zarr(f\"s3://carbonplan-climatetrace/validation/harris_3000m_agbd.zarr\")\n", + "harris = xr.open_zarr(\"s3://carbonplan-climatetrace/validation/harris_3000m_agbd.zarr\")\n", "harris = harris.rename({\"lon\": \"x\", \"lat\": \"y\"})\n", "harris.attrs[\"crs\"] = \"EPSG:4326\"\n", "harris" @@ -422,7 +417,7 @@ "metadata": {}, "outputs": [], "source": [ - "output_filename = f\"s3://carbonplan-climatetrace/validation/globbiomass_3000m_agbd.zarr\"\n", + "output_filename = \"s3://carbonplan-climatetrace/validation/globbiomass_3000m_agbd.zarr\"\n", "ds.drop(\"spatial_ref\").to_zarr(output_filename, consolidated=True)" ] }, @@ -716,7 +711,7 @@ "metadata": {}, "outputs": [], "source": [ - "spawn = xr.open_zarr(f\"s3://carbonplan-climatetrace/validation/spawn_3000m_agbd.zarr\")\n", + "spawn = xr.open_zarr(\"s3://carbonplan-climatetrace/validation/spawn_3000m_agbd.zarr\")\n", "spawn = spawn.rio.write_crs(\"EPSG:4326\")" ] }, @@ -759,7 +754,7 @@ "outputs": [], "source": [ "land_mask.to_zarr(\n", - " f\"s3://carbonplan-climatetrace/validation/land_mask.zarr\", consolidated=True, mode=\"w\"\n", + " \"s3://carbonplan-climatetrace/validation/land_mask.zarr\", consolidated=True, mode=\"w\"\n", ")" ] }, @@ -778,8 +773,8 @@ "metadata": {}, "outputs": [], "source": [ - "from s3fs import S3FileSystem\n", "import geopandas\n", + "from s3fs import S3FileSystem\n", "\n", "fs = S3FileSystem()\n", "with fs.open(f\"s3://carbonplan-climatetrace/{version}/masks/valid_landsat.shp.zip\") as f:\n", @@ -833,7 +828,7 @@ "metadata": {}, "outputs": [], "source": [ - "spawn = xr.open_zarr(f\"s3://carbonplan-climatetrace/validation/spawn_3000m_agbd.zarr\")\n", + "spawn = xr.open_zarr(\"s3://carbonplan-climatetrace/validation/spawn_3000m_agbd.zarr\")\n", "spawn = spawn.rio.write_crs(\"EPSG:4326\")" ] }, @@ -895,7 +890,7 @@ "outputs": [], "source": [ "ar6_mask.to_zarr(\n", - " f\"s3://carbonplan-climatetrace/validation/ar6_mask.zarr\", consolidated=True, mode=\"w\"\n", + " \"s3://carbonplan-climatetrace/validation/ar6_mask.zarr\", consolidated=True, mode=\"w\"\n", ")" ] }, @@ -914,7 +909,7 @@ "metadata": {}, "outputs": [], "source": [ - "spawn = xr.open_zarr(f\"s3://carbonplan-climatetrace/validation/spawn_3000m_agbd.zarr\")\n", + "spawn = xr.open_zarr(\"s3://carbonplan-climatetrace/validation/spawn_3000m_agbd.zarr\")\n", "spawn = spawn.rio.write_crs(\"EPSG:4326\")" ] }, @@ -971,7 +966,7 @@ "outputs": [], "source": [ "realm_mask.to_zarr(\n", - " f\"s3://carbonplan-climatetrace/validation/realm_mask.zarr\", consolidated=True, mode=\"w\"\n", + " \"s3://carbonplan-climatetrace/validation/realm_mask.zarr\", consolidated=True, mode=\"w\"\n", ")" ] }, diff --git a/notebooks/analysis/get_landsat_data_availability.ipynb b/notebooks/analysis/get_landsat_data_availability.ipynb index 440f8a2..ac8c5a4 100644 --- a/notebooks/analysis/get_landsat_data_availability.ipynb +++ b/notebooks/analysis/get_landsat_data_availability.ipynb @@ -10,31 +10,18 @@ "%load_ext autoreload\n", "%autoreload 2\n", "\n", - "import dask\n", + "import geopandas as gpd\n", "import numpy as np\n", - "import pandas as pd\n", - "import xarray as xr\n", - "\n", - "from carbonplan_trace.tiles import tiles\n", - "from carbonplan_trace.v1 import biomass_rollup\n", - "\n", - "import fsspec\n", - "import h5py\n", "import regionmask\n", - "from rasterio.session import AWSSession\n", - "from rasterio.warp import Resampling\n", + "from s3fs import S3FileSystem\n", "\n", + "from carbonplan_trace.tiles import tiles\n", "from carbonplan_trace.v1 import utils\n", - "from collections import defaultdict\n", "from carbonplan_trace.v1.landsat_preprocess import (\n", " access_credentials,\n", " find_months_of_interest,\n", " make_datestamps,\n", - ")\n", - "from carbonplan_trace.v0.core import compute_grid_area\n", - "import rioxarray as rio\n", - "import geopandas as gpd\n", - "from s3fs import S3FileSystem" + ")" ] }, { @@ -87,10 +74,7 @@ "metadata": {}, "outputs": [], "source": [ - "import boto3\n", - "from rasterio.session import AWSSession\n", - "from s3fs import S3FileSystem\n", - "from carbonplan_trace.v1.landsat_preprocess import scene_seasonal_average" + "from s3fs import S3FileSystem" ] }, { @@ -224,7 +208,7 @@ "outputs": [], "source": [ "valid_scene_ids = list(\n", - " set([\"{:03d}/{:03d}\".format(path, row) for path, row in scenes_with_valid_data])\n", + " {f\"{path:03d}/{row:03d}\" for path, row in scenes_with_valid_data}\n", ")" ] }, diff --git a/notebooks/analysis/temporal_variation.ipynb b/notebooks/analysis/temporal_variation.ipynb index 80ca16d..e9e2335 100644 --- a/notebooks/analysis/temporal_variation.ipynb +++ b/notebooks/analysis/temporal_variation.ipynb @@ -14,17 +14,12 @@ "\n", "fs = S3FileSystem()\n", "\n", + "import copy\n", "import random\n", - "import rasterio as rio\n", + "\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import xarray as xr\n", - "import dask\n", - "import copy\n", - "import scipy\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "from datetime import datetime\n", - "import time\n", "\n", "from carbonplan_trace.v1 import emissions_workflow" ] @@ -569,7 +564,7 @@ " ax1.set_ylabel(\"AGB (Mg/ha)\", color=\"b\")\n", " ax2.set_ylabel(\"Z Score Flux\", color=\"r\")\n", " ax2.set_yticks([-2.5, -1.96, -1, 0, 1, 1.96, 2.5])\n", - " print(f\"overall pvalue =\", result[\"pvalue\"])\n", + " print(\"overall pvalue =\", result[\"pvalue\"])\n", " plt.show()\n", " plt.close()" ] @@ -625,8 +620,8 @@ "outputs": [], "source": [ "plot_ts_diff(subset)\n", - "plt.suptitle(f\"Original Flux\", y=1)\n", - "plt.savefig(f\"original_flux.png\")\n", + "plt.suptitle(\"Original Flux\", y=1)\n", + "plt.savefig(\"original_flux.png\")\n", "plt.show()" ] }, @@ -639,8 +634,8 @@ "source": [ "# 95%\n", "plot_ts_diff(subset_result)\n", - "plt.suptitle(f\"Smoothed Flux\", y=1)\n", - "plt.savefig(f\"smoothed_flux.png\")\n", + "plt.suptitle(\"Smoothed Flux\", y=1)\n", + "plt.savefig(\"smoothed_flux.png\")\n", "plt.show()" ] }, @@ -653,8 +648,8 @@ "source": [ "# 90%\n", "plot_ts_diff(subset_result)\n", - "plt.suptitle(f\"Smoothed Flux\", y=1)\n", - "plt.savefig(f\"smoothed_flux.png\")\n", + "plt.suptitle(\"Smoothed Flux\", y=1)\n", + "plt.savefig(\"smoothed_flux.png\")\n", "plt.show()" ] }, @@ -673,8 +668,7 @@ "metadata": {}, "outputs": [], "source": [ - "from sklearn import decomposition\n", - "from numpy.random import RandomState" + "from sklearn import decomposition" ] }, { diff --git a/notebooks/analysis/v1_article_figures_stats.ipynb b/notebooks/analysis/v1_article_figures_stats.ipynb index 8b236f8..dcc8a64 100644 --- a/notebooks/analysis/v1_article_figures_stats.ipynb +++ b/notebooks/analysis/v1_article_figures_stats.ipynb @@ -7,10 +7,9 @@ "metadata": {}, "outputs": [], "source": [ - "import xarray as xr\n", - "from numcodecs.zlib import Zlib\n", "import pandas as pd\n", - "import numpy as np" + "import xarray as xr\n", + "from numcodecs.zlib import Zlib" ] }, { @@ -336,10 +335,11 @@ "metadata": {}, "outputs": [], "source": [ - "from s3fs import S3FileSystem\n", "import geopandas\n", - "from carbonplan_trace.v1 import utils\n", "import regionmask\n", + "from s3fs import S3FileSystem\n", + "\n", + "from carbonplan_trace.v1 import utils\n", "\n", "fs = S3FileSystem()\n", "with fs.open(f\"s3://carbonplan-climatetrace/{version}/masks/valid_landsat.shp.zip\") as f:\n", diff --git a/notebooks/analysis/validate_biomass.ipynb b/notebooks/analysis/validate_biomass.ipynb index bd03c2a..501820d 100644 --- a/notebooks/analysis/validate_biomass.ipynb +++ b/notebooks/analysis/validate_biomass.ipynb @@ -10,21 +10,21 @@ "%load_ext autoreload\n", "%autoreload 2\n", "\n", - "import xarray as xr\n", - "from matplotlib import cm\n", "import cartopy.crs as ccrs\n", "import matplotlib.pyplot as plt\n", + "import xarray as xr\n", "from carbonplan_styles.mpl import set_theme\n", + "from matplotlib import cm\n", "\n", "set_theme()\n", - "from sklearn.metrics import r2_score, mean_absolute_error, mean_absolute_percentage_error\n", "import numpy as np\n", - "import seaborn as sns\n", "import pandas as pd\n", - "from carbonplan_trace.tiles import tiles\n", - "from carbonplan_trace.v0.core import coarsen_emissions\n", + "import rasterio\n", + "import seaborn as sns\n", "from carbonplan_data.utils import set_zarr_encoding\n", - "import rasterio" + "from sklearn.metrics import mean_absolute_error, r2_score\n", + "\n", + "from carbonplan_trace.v0.core import coarsen_emissions" ] }, { @@ -42,9 +42,10 @@ "metadata": {}, "outputs": [], "source": [ - "from carbonplan_trace.v1.landsat_preprocess import access_credentials\n", "import fsspec\n", "\n", + "from carbonplan_trace.v1.landsat_preprocess import access_credentials\n", + "\n", "access_key_id, secret_access_key = access_credentials()\n", "fs = fsspec.get_filesystem_class(\"s3\")(\n", " key=access_key_id,\n", @@ -167,7 +168,7 @@ "metadata": {}, "outputs": [], "source": [ - "v0_emission = xr.open_zarr(f\"s3://carbonplan-climatetrace/v0.4/global/3000m/raster_tot.zarr\")\n", + "v0_emission = xr.open_zarr(\"s3://carbonplan-climatetrace/v0.4/global/3000m/raster_tot.zarr\")\n", "v0_emission10km = coarsen_emissions(\n", " v0_emission, factor=coarsening_factor, mask_var=\"emissions\", method=\"mean\"\n", ").rename({\"year\": \"time\"})\n", @@ -264,7 +265,7 @@ "realms = realms.assign_coords({\"lat\": realms.lat.round(4), \"lon\": realms.lon.round(4)})\n", "ds[\"realm\"] = realms.realm\n", "\n", - "ar6_mask = xr.open_zarr(f\"s3://carbonplan-climatetrace/validation/ar6_mask.zarr\")\n", + "ar6_mask = xr.open_zarr(\"s3://carbonplan-climatetrace/validation/ar6_mask.zarr\")\n", "ar6_mask = ar6_mask.assign_coords({\"lat\": ar6_mask.lat.round(4), \"lon\": ar6_mask.lon.round(4)})\n", "ds[\"ar6_mask\"] = ar6_mask.ar6_mask" ] @@ -297,8 +298,8 @@ "metadata": {}, "outputs": [], "source": [ - "from cartopy.io import shapereader\n", "import geopandas as gpd\n", + "from cartopy.io import shapereader\n", "\n", "\n", "def cartopy_proj_plate_carree():\n", @@ -325,7 +326,6 @@ "outputs": [], "source": [ "import matplotlib as mpl\n", - "from mpl_toolkits.axes_grid1 import make_axes_locatable\n", "\n", "\n", "def map_pretty(ax, title=\"\", min_lat=-90, max_lat=90, min_lon=-180, max_lon=180):\n", @@ -364,7 +364,6 @@ " cbar_label=\"\",\n", " cmap=\"viridis\",\n", "):\n", - "\n", " cax = fig.add_axes([x_location, y_location, width, height])\n", " cax.text(\n", " 0.5,\n", diff --git a/notebooks/analysis/validate_biomass_training_data.ipynb b/notebooks/analysis/validate_biomass_training_data.ipynb index c309da3..984e1f1 100644 --- a/notebooks/analysis/validate_biomass_training_data.ipynb +++ b/notebooks/analysis/validate_biomass_training_data.ipynb @@ -9,27 +9,26 @@ "source": [ "%load_ext autoreload\n", "%autoreload 2\n", + "import warnings\n", + "\n", + "import cartopy.crs as ccrs\n", "import fsspec\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import xarray as xr\n", - "from carbonplan.data import cat\n", - "import cartopy.crs as ccrs\n", - "import matplotlib.pyplot as plt\n", - "import warnings\n", "\n", "warnings.filterwarnings('ignore')\n", + "import os\n", + "\n", "import matplotlib\n", "from carbonplan_data import utils\n", "from matplotlib import cm\n", - "from sklearn.metrics import r2_score, mean_absolute_error, mean_absolute_percentage_error\n", - "import rioxarray\n", + "from sklearn.metrics import mean_absolute_error, r2_score\n", "\n", - "from carbonplan_trace.v1 import utils as trace_utils\n", - "from carbonplan_trace.v1 import glas_allometric_eq as allo\n", - "\n", - "import os\n", "from carbonplan_trace.v0.data import cat as trace_cat\n", + "from carbonplan_trace.v1 import glas_allometric_eq as allo\n", + "from carbonplan_trace.v1 import utils as trace_utils\n", "\n", "matplotlib.rc('font', family='sans-serif')\n", "matplotlib.rc('font', serif='Helvetica Neue')\n", @@ -796,8 +795,8 @@ "metadata": {}, "outputs": [], "source": [ - "from cartopy.io import shapereader\n", "import geopandas as gpd\n", + "from cartopy.io import shapereader\n", "\n", "\n", "def cartopy_proj_albers():\n", @@ -850,7 +849,6 @@ "outputs": [], "source": [ "import matplotlib as mpl\n", - "from mpl_toolkits.axes_grid1 import make_axes_locatable\n", "\n", "\n", "def map_pretty(ax, title=\"\", min_lat=-90, max_lat=90, min_lon=-180, max_lon=180):\n", @@ -889,7 +887,6 @@ " cbar_label=\"\",\n", " cmap=\"viridis\",\n", "):\n", - "\n", " cax = fig.add_axes([x_location, y_location, width, height])\n", " cax.text(\n", " 0.5,\n", diff --git a/notebooks/analysis/validate_change_point_dection.ipynb b/notebooks/analysis/validate_change_point_dection.ipynb index 8116bf4..2b67c1d 100644 --- a/notebooks/analysis/validate_change_point_dection.ipynb +++ b/notebooks/analysis/validate_change_point_dection.ipynb @@ -30,20 +30,14 @@ "%load_ext autoreload\n", "%autoreload 2\n", "\n", - "import xarray as xr\n", - "from matplotlib import cm\n", - "import cartopy.crs as ccrs\n", "import matplotlib.pyplot as plt\n", "from carbonplan_styles.mpl import set_theme\n", "\n", "set_theme()\n", - "from sklearn.metrics import r2_score, mean_absolute_error, mean_absolute_percentage_error\n", "import numpy as np\n", - "import seaborn as sns\n", "import pandas as pd\n", - "from carbonplan_trace.tiles import tiles\n", - "from carbonplan_trace.v0.core import coarsen_emissions\n", - "from carbonplan_data.utils import set_zarr_encoding" + "\n", + "from carbonplan_trace.tiles import tiles" ] }, { @@ -54,6 +48,7 @@ "outputs": [], "source": [ "import random\n", + "\n", "from carbonplan_trace.v1.biomass_rollup import open_biomass_tile\n", "\n", "random.seed(1)" diff --git a/notebooks/analysis/validate_emissions.ipynb b/notebooks/analysis/validate_emissions.ipynb index 4a9f4ff..55f8dbc 100644 --- a/notebooks/analysis/validate_emissions.ipynb +++ b/notebooks/analysis/validate_emissions.ipynb @@ -26,10 +26,8 @@ "outputs": [], "source": [ "import geopandas\n", - "import pandas as pd\n", - "from io import StringIO\n", "import matplotlib.pyplot as plt\n", - "\n", + "import pandas as pd\n", "from carbonplan_styles.mpl import set_theme\n", "\n", "set_theme()" diff --git a/notebooks/analysis/validate_height_metric.ipynb b/notebooks/analysis/validate_height_metric.ipynb index 1357ce8..8ef624a 100644 --- a/notebooks/analysis/validate_height_metric.ipynb +++ b/notebooks/analysis/validate_height_metric.ipynb @@ -9,24 +9,21 @@ "source": [ "%load_ext autoreload\n", "%autoreload 2\n", + "import warnings\n", + "\n", "import fsspec\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import xarray as xr\n", - "from carbonplan.data import cat\n", - "import cartopy.crs as ccrs\n", - "import matplotlib.pyplot as plt\n", - "import warnings\n", "\n", "warnings.filterwarnings('ignore')\n", + "import os\n", + "\n", "import matplotlib\n", - "from carbonplan_data import utils\n", - "from matplotlib import cm\n", "\n", - "from carbonplan_trace.v1 import utils as trace_utils\n", - "import os\n", - "from carbonplan_trace.v0.data import cat as trace_cat\n", "from carbonplan_trace.v1 import glas_height_metrics as ht\n", + "from carbonplan_trace.v1 import utils as trace_utils\n", "\n", "matplotlib.rc('font', family='sans-serif')\n", "matplotlib.rc('font', serif='Helvetica Neue')\n", @@ -236,7 +233,6 @@ "outputs": [], "source": [ "def merge_ds(study, ref, var_name, precision=3):\n", - "\n", " variables_study = [\n", " var for var in list(study.variables.keys()) if var_name.lower() in var.lower()\n", " ]\n", @@ -383,7 +379,7 @@ "metadata": {}, "outputs": [], "source": [ - "from sklearn.metrics import r2_score, mean_squared_error\n", + "from sklearn.metrics import mean_squared_error, r2_score\n", "\n", "\n", "def plot_scatter_comparison(ax, df, col_name, compared_col_name, params):\n", @@ -437,6 +433,8 @@ "outputs": [], "source": [ "# calculate QMCH in a few different ways\n", + "\n", + "\n", "def quadratic_mean_to_ground_ht(ds):\n", " \"\"\"\n", " Quadratic mean height of the waveform from ground peak to signal beginning (meters).\n", @@ -470,6 +468,8 @@ "outputs": [], "source": [ "# calculate MeanH in a few different ways\n", + "\n", + "\n", "def mean_to_ground_ht(ds):\n", " \"\"\"\n", " Quadratic mean height of the waveform from ground peak to signal beginning (meters).\n", @@ -526,7 +526,6 @@ "}\n", "\n", "for var in [\"QMCH\", \"MeanH\"]:\n", - "\n", " joined = merge_ds(study=sub, ref=margolis, var_name=var, precision=3)\n", "\n", " plt.figure(figsize=(13, 4.5))\n", @@ -823,9 +822,9 @@ " following linear transformation in order to calculate on the same scale as data published by Margolis et al. (2015)\n", " senergy = -4.397006 + 0.006208 * senergy_whrc\n", " \"\"\"\n", - " from carbonplan_trace.v1.glas_preprocess import (\n", + " from carbonplan_trace.v1.glas_preprocess import ( # avoid circular import\n", " select_valid_area,\n", - " ) # avoid circular import\n", + " )\n", "\n", " path = \"gs://carbonplan-climatetrace/inputs/volt_table.csv\"\n", " volt_table = pd.read_csv(path)\n", @@ -862,10 +861,10 @@ " following linear transformation in order to calculate on the same scale as data published by Margolis et al. (2015)\n", " senergy = -4.397006 + 0.006208 * senergy_whrc\n", " \"\"\"\n", - " from carbonplan_trace.v1.glas_preprocess import (\n", + " from carbonplan_trace.v1.glas_preprocess import ( # avoid circular import\n", " select_valid_area,\n", " smooth_wf,\n", - " ) # avoid circular import\n", + " )\n", "\n", " path = \"gs://carbonplan-climatetrace/inputs/volt_table.csv\"\n", " volt_table = pd.read_csv(path)\n", @@ -914,10 +913,9 @@ " following linear transformation in order to calculate on the same scale as data published by Margolis et al. (2015)\n", " senergy = -4.397006 + 0.006208 * senergy_whrc\n", " \"\"\"\n", - " from carbonplan_trace.v1.glas_preprocess import (\n", + " from carbonplan_trace.v1.glas_preprocess import ( # avoid circular import\n", " select_valid_area,\n", - " smooth_wf,\n", - " ) # avoid circular import\n", + " )\n", "\n", " ds = ht.get_dist_metric_value(ds, metric=\"adj_ground_peak_dist\")\n", " # the processed wf is from sig beg to sig end, select adj ground peak to sig end instead\n", @@ -985,9 +983,9 @@ " following linear transformation in order to calculate on the same scale as data published by Margolis et al. (2015)\n", " senergy = -4.397006 + 0.006208 * senergy_whrc\n", " \"\"\"\n", - " from carbonplan_trace.v1.glas_preprocess import (\n", + " from carbonplan_trace.v1.glas_preprocess import ( # avoid circular import\n", " select_valid_area,\n", - " ) # avoid circular import\n", + " )\n", "\n", " path = \"gs://carbonplan-climatetrace/inputs/volt_table.csv\"\n", " volt_table = pd.read_csv(path)\n", @@ -1028,6 +1026,7 @@ "# study['senergy_smooth'] = energy_adj_ground_to_sig_end_smooth(study)\n", "# study['senergy_no_transform'] = energy_adj_ground_to_sig_end_no_transform(study)\n", "\n", + "\n", "study[\"senergy_new_adj\"] = energy_adj_ground_to_sig_end_new_adj(study)" ] }, diff --git a/notebooks/analysis/validate_v1.ipynb b/notebooks/analysis/validate_v1.ipynb index 19d6b9e..14e328a 100644 --- a/notebooks/analysis/validate_v1.ipynb +++ b/notebooks/analysis/validate_v1.ipynb @@ -25,16 +25,11 @@ "metadata": {}, "outputs": [], "source": [ - "import geopandas\n", - "import pandas as pd\n", - "from io import StringIO\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import urllib3\n", "import xarray as xr\n", "from carbonplan_styles.mpl import set_theme\n", - "from carbonplan_styles.colors import colors\n", - "from carbonplan_trace.v1.emissions_workflow import open_fire_mask\n", - "import urllib3\n", - "import numpy as np\n", "\n", "urllib3.disable_warnings()\n", "set_theme()" diff --git a/notebooks/blogpost_sample_notebook.ipynb b/notebooks/blogpost_sample_notebook.ipynb index 7ed73f8..0747f6d 100644 --- a/notebooks/blogpost_sample_notebook.ipynb +++ b/notebooks/blogpost_sample_notebook.ipynb @@ -22,14 +22,12 @@ "metadata": {}, "outputs": [], "source": [ - "import xarray as xr\n", - "import pandas as pd\n", - "from carbonplan_trace.v0.data import cat, cat_file\n", - "\n", - "from carbonplan_styles.colors import colors\n", - "from carbonplan_styles.mpl import set_theme\n", "import cartopy.crs as ccrs\n", "import matplotlib.pyplot as plt\n", + "from carbonplan_styles.colors import colors\n", + "from carbonplan_styles.mpl import set_theme\n", + "\n", + "from carbonplan_trace.v0.data import cat\n", "\n", "theme = \"dark\"\n", "set_theme(style=f\"carbonplan_{theme}\")\n", diff --git a/notebooks/lidar_blog_sample_notebook.ipynb b/notebooks/lidar_blog_sample_notebook.ipynb index 7147b9b..dd157e6 100644 --- a/notebooks/lidar_blog_sample_notebook.ipynb +++ b/notebooks/lidar_blog_sample_notebook.ipynb @@ -23,19 +23,14 @@ "outputs": [], "source": [ "import fsspec\n", + "import matplotlib.pyplot as plt\n", "import xarray as xr\n", - "import pandas as pd\n", - "from carbonplan_trace.v0.data import cat, cat_file\n", + "from carbonplan_styles.colors import colors\n", + "from carbonplan_styles.mpl import set_theme\n", "\n", "import carbonplan_trace.v1.utils as utils\n", - "from carbonplan_trace.v1.glas_preprocess import preprocess\n", - "from carbonplan_trace.v1.glas_height_metrics import plot_shot\n", "from carbonplan_trace.v1.glas_allometric_eq import apply_allometric_equation\n", - "\n", - "from carbonplan_styles.colors import colors\n", - "from carbonplan_styles.mpl import set_theme\n", - "import cartopy.crs as ccrs\n", - "import matplotlib.pyplot as plt\n", + "from carbonplan_trace.v1.glas_height_metrics import plot_shot\n", "\n", "theme = \"dark\"\n", "set_theme(style=f\"carbonplan_{theme}\")\n", diff --git a/notebooks/processing/apply_allometric_eq.ipynb b/notebooks/processing/apply_allometric_eq.ipynb index d196aae..474ac22 100644 --- a/notebooks/processing/apply_allometric_eq.ipynb +++ b/notebooks/processing/apply_allometric_eq.ipynb @@ -16,18 +16,15 @@ "metadata": {}, "outputs": [], "source": [ - "import fsspec\n", - "import time\n", "import json\n", - "import numpy as np\n", + "\n", + "import fsspec\n", "import xarray as xr\n", - "import matplotlib.pyplot as plt\n", "\n", - "from carbonplan_trace.v1.glas_preprocess import preprocess\n", "import carbonplan_trace.v1.glas_allometric_eq as allo\n", "import carbonplan_trace.v1.utils as utils\n", "from carbonplan_trace.v1.glas_height_metrics import get_all_height_metrics\n", - "from carbonplan_trace.utils import zarr_is_complete" + "from carbonplan_trace.v1.glas_preprocess import preprocess" ] }, { @@ -37,6 +34,7 @@ "outputs": [], "source": [ "from s3fs import S3FileSystem\n", + "\n", "from carbonplan_trace.v1.landsat_preprocess import access_credentials\n", "\n", "access_key_id, secret_access_key = access_credentials()\n", @@ -122,7 +120,6 @@ "outputs": [], "source": [ "import carbonplan_trace.v1.glas_height_metrics as h\n", - "from carbonplan_trace.v1.glas_preprocess import get_modeled_waveform\n", "\n", "lat_tag = \"50N\"\n", "lon_tag = \"080W\"\n", @@ -385,7 +382,7 @@ " for lat, lon in all_lat_lon_tags:\n", " fn = f\"{lat}_{lon}\"\n", " output_path = f\"s3://carbonplan-climatetrace/v1/biomass/{lat}_{lon}.zarr/.zmetadata\"\n", - " if not fs.exists(output_path) and not fn in no_data_tiles:\n", + " if not fs.exists(output_path) and fn not in no_data_tiles:\n", " lat_lon_tags.append((lat, lon))\n", "\n", " return lat_lon_tags\n", diff --git a/notebooks/processing/change_point_detection.ipynb b/notebooks/processing/change_point_detection.ipynb index e375ffc..07458d6 100644 --- a/notebooks/processing/change_point_detection.ipynb +++ b/notebooks/processing/change_point_detection.ipynb @@ -10,7 +10,6 @@ "%load_ext autoreload\n", "%autoreload 2\n", "\n", - "from pyproj import CRS\n", "import boto3\n", "from rasterio.session import AWSSession\n", "from s3fs import S3FileSystem\n", @@ -18,25 +17,17 @@ "aws_session = AWSSession(boto3.Session(), requester_pays=True) # profile_name='default'),\n", "fs = S3FileSystem(requester_pays=True)\n", "\n", - "import rasterio as rio\n", - "import numpy as np\n", - "import xarray as xr\n", - "import dask\n", - "import os\n", - "import fsspec\n", - "\n", - "import rioxarray # for the extension to load\n", - "import pandas as pd\n", + "from datetime import datetime\n", "\n", + "import dask\n", + "import numpy as np\n", "from dask_gateway import Gateway\n", - "from carbonplan_trace.v1.landsat_preprocess import access_credentials, test_credentials\n", - "from carbonplan_trace.v1 import utils, load\n", - "from carbonplan_trace.tiles import tiles\n", - "import prefect\n", - "from prefect import task, Flow, Parameter\n", + "from prefect import Flow, task\n", "from prefect.executors import DaskExecutor\n", - "from prefect.utilities.debug import raise_on_exception\n", - "from datetime import datetime" + "\n", + "from carbonplan_trace.tiles import tiles\n", + "from carbonplan_trace.v1 import utils\n", + "from carbonplan_trace.v1.landsat_preprocess import access_credentials" ] }, { @@ -46,8 +37,8 @@ "metadata": {}, "outputs": [], "source": [ - "from carbonplan_trace.v1 import postprocess as p\n", - "from carbonplan_trace.v1 import change_point_detection as c" + "from carbonplan_trace.v1 import change_point_detection as c\n", + "from carbonplan_trace.v1 import postprocess as p" ] }, { @@ -230,12 +221,11 @@ "\n", " for lat_increment in np.arange(0, 10, tile_degree_size):\n", " for lon_increment in np.arange(0, 10, tile_degree_size):\n", - " task_tag = \"{}_{}_{}_{}\".format(min_lat, min_lon, lat_increment, lon_increment)\n", + " task_tag = f\"{min_lat}_{min_lon}_{lat_increment}_{lon_increment}\"\n", " if task_tag in completed_subtiles:\n", " # if this subtile has already been ran, continue\n", " continue\n", " else:\n", - "\n", " increment_parameters = prefect_parameters.copy()\n", " increment_parameters[\"LAT_INCREMENT\"] = lat_increment\n", " increment_parameters[\"LON_INCREMENT\"] = lon_increment\n", @@ -304,7 +294,7 @@ " if new_state.is_running():\n", " print(\"running!\")\n", " if new_state.is_failed():\n", - " print(\"this task {} failed\".format(task))\n", + " print(f\"this task {task} failed\")\n", " raise ValueError(\"OH NO\") # function that sends a notification\n", " return new_state" ] diff --git a/notebooks/processing/download_srtm.ipynb b/notebooks/processing/download_srtm.ipynb index de1d8bd..7044bc1 100644 --- a/notebooks/processing/download_srtm.ipynb +++ b/notebooks/processing/download_srtm.ipynb @@ -7,24 +7,18 @@ "metadata": {}, "outputs": [], "source": [ + "import netrc\n", "import os\n", + "from urllib.error import HTTPError, URLError\n", + "from urllib.parse import urlparse\n", + "\n", "import dask\n", "import fsspec\n", - "import netrc\n", - "from urllib.parse import urlparse\n", - "from urllib.error import HTTPError, URLError\n", - "import rioxarray\n", "import xarray as xr\n", "\n", "URS_URL = \"https://urs.earthdata.nasa.gov\"\n", "import base64\n", - "from urllib.request import (\n", - " urlopen,\n", - " Request,\n", - " build_opener,\n", - " HTTPCookieProcessor,\n", - ")\n", - "\n", + "from urllib.request import HTTPCookieProcessor, Request, build_opener\n", "\n", "from tqdm import tqdm" ] @@ -85,8 +79,8 @@ " username, account, password = info.authenticators(urlparse(URS_URL).hostname)\n", " errprefix = \"netrc error: \"\n", " except Exception as e:\n", - " if not (\"No such file\" in str(e)):\n", - " print(\"netrc error: {0}\".format(str(e)))\n", + " if \"No such file\" not in str(e):\n", + " print(f\"netrc error: {str(e)}\")\n", " username = None\n", " password = None\n", "\n", @@ -94,13 +88,13 @@ " if not username:\n", " username = get_username()\n", " password = get_password()\n", - " credentials = \"{0}:{1}\".format(username, password)\n", + " credentials = f\"{username}:{password}\"\n", " credentials = base64.b64encode(credentials.encode(\"ascii\")).decode(\"ascii\")\n", "\n", " if url:\n", " try:\n", " req = Request(url)\n", - " req.add_header(\"Authorization\", \"Basic {0}\".format(credentials))\n", + " req.add_header(\"Authorization\", f\"Basic {credentials}\")\n", " opener = build_opener(HTTPCookieProcessor())\n", " opener.open(req)\n", " except HTTPError:\n", @@ -144,7 +138,7 @@ " try:\n", " req = Request(url)\n", " if credentials:\n", - " req.add_header(\"Authorization\", \"Basic {0}\".format(credentials))\n", + " req.add_header(\"Authorization\", f\"Basic {credentials}\")\n", " opener = build_opener(HTTPCookieProcessor())\n", "\n", " with fsspec.open(target_url, mode=\"wb\") as target:\n", @@ -153,11 +147,11 @@ " out.append(target_url)\n", "\n", " except HTTPError as e:\n", - " print(\"HTTPError {0}, {1}\".format(e.code, e.reason), filename)\n", + " print(f\"HTTPError {e.code}, {e.reason}\", filename)\n", " except URLError as e:\n", - " print(\"URLError: {0}\".format(e.reason), filename)\n", - " except IOError:\n", - " print(\"IOError: {0}\".format(e.reason), filename)\n", + " print(f\"URLError: {e.reason}\", filename)\n", + " except OSError:\n", + " print(f\"IOError: {e.reason}\", filename)\n", " except KeyboardInterrupt:\n", " quit()\n", " except:\n", diff --git a/notebooks/processing/inference.ipynb b/notebooks/processing/inference.ipynb index a001971..ed98c10 100644 --- a/notebooks/processing/inference.ipynb +++ b/notebooks/processing/inference.ipynb @@ -18,36 +18,23 @@ "%load_ext autoreload\n", "%autoreload 2\n", "\n", - "from pyproj import CRS\n", "import boto3\n", "from rasterio.session import AWSSession\n", "from s3fs import S3FileSystem\n", "\n", "aws_session = AWSSession(boto3.Session(), requester_pays=True) # profile_name='default'),\n", "fs = S3FileSystem(requester_pays=True) # profile='default',\n", - "import xgboost as xgb\n", "\n", - "from osgeo.gdal import VSICurlClearCache\n", - "import rasterio as rio\n", - "import numpy as np\n", - "import xarray as xr\n", "import dask\n", - "import os\n", - "import fsspec\n", "import geopandas as gpd\n", - "import rioxarray # for the extension to load\n", - "import matplotlib.pyplot as plt\n", - "import utm\n", + "import numpy as np\n", "import pandas as pd\n", - "from datetime import datetime\n", - "import time\n", - "import json\n", - "import zarr\n", - "import awswrangler as wr\n", + "import rasterio as rio\n", "from dask_gateway import Gateway\n", - "from carbonplan_trace.v1.landsat_preprocess import access_credentials, test_credentials\n", + "\n", + "from carbonplan_trace.v1 import utils\n", "from carbonplan_trace.v1.inference import predict, predict_delayed\n", - "from carbonplan_trace.v1 import utils" + "from carbonplan_trace.v1.landsat_preprocess import access_credentials, test_credentials" ] }, { @@ -196,8 +183,6 @@ "metadata": {}, "outputs": [], "source": [ - "from carbonplan_trace.v1.glas_allometric_eq import REALM_GROUPINGS\n", - "\n", "processed_scenes = []\n", "for year in np.arange(2014, 2021):\n", " processed_scenes.extend(fs.ls(f\"{bucket}/inference/rf/{year}\", recursive=True))\n", diff --git a/notebooks/processing/model.ipynb b/notebooks/processing/model.ipynb index 310d5c0..56ef4cd 100644 --- a/notebooks/processing/model.ipynb +++ b/notebooks/processing/model.ipynb @@ -18,14 +18,14 @@ "metadata": {}, "outputs": [], "source": [ - "from carbonplan_trace.v1.glas_allometric_eq import REALM_GROUPINGS\n", - "from carbonplan_trace.v1 import load\n", - "import carbonplan_trace.v1.model as m\n", - "import pandas as pd\n", - "from carbonplan_trace.v1.landsat_preprocess import access_credentials\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", + "import pandas as pd\n", "\n", - "import matplotlib.pyplot as plt" + "import carbonplan_trace.v1.model as m\n", + "from carbonplan_trace.v1 import load\n", + "from carbonplan_trace.v1.glas_allometric_eq import REALM_GROUPINGS\n", + "from carbonplan_trace.v1.landsat_preprocess import access_credentials" ] }, { @@ -110,7 +110,6 @@ "\n", "\n", "def get_all_prediction_result(model, df_train, df_test, df_val):\n", - "\n", " df_train[\"biomass_pred\"] = model._predict(df_train)\n", " df_test[\"biomass_pred\"] = model._predict(df_test)\n", " df_val[\"biomass_pred\"] = model._predict(df_val)\n", @@ -687,7 +686,6 @@ "\n", " # this for loop is for running different parameter sets in HPO\n", " for params in [{}]:\n", - "\n", " # instantiating the model also does .fit\n", " # this will load the model if it already exist and overwrite=False, and fit the model if overwrite=True or the model does not exist\n", " model = model_class(\n", diff --git a/notebooks/processing/postprocess.ipynb b/notebooks/processing/postprocess.ipynb index 9959aa5..10fb589 100644 --- a/notebooks/processing/postprocess.ipynb +++ b/notebooks/processing/postprocess.ipynb @@ -35,38 +35,24 @@ "%load_ext autoreload\n", "%autoreload 2\n", "\n", - "from pyproj import CRS\n", "import boto3\n", "from rasterio.session import AWSSession\n", "from s3fs import S3FileSystem\n", "\n", "aws_session = AWSSession(boto3.Session(), requester_pays=True) # profile_name='default'),\n", "fs = S3FileSystem(requester_pays=True)\n", - "import xgboost as xgb\n", "\n", - "from osgeo.gdal import VSICurlClearCache\n", - "import rasterio as rio\n", - "import numpy as np\n", - "import xarray as xr\n", - "import dask\n", - "import os\n", - "import fsspec\n", - "\n", - "import rioxarray # for the extension to load\n", - "import pandas as pd\n", "from datetime import datetime\n", "\n", + "import dask\n", + "import numpy as np\n", "from dask_gateway import Gateway\n", - "from carbonplan_trace.v1.landsat_preprocess import access_credentials, test_credentials\n", - "from carbonplan_trace.v1.inference import predict, predict_delayed\n", - "from carbonplan_trace.v1 import utils, postprocess, load\n", - "from carbonplan_trace.tiles import tiles\n", - "from carbonplan_trace.v1.landsat_preprocess import access_credentials, test_credentials\n", - "import prefect\n", - "from prefect import task, Flow, Parameter\n", + "from prefect import Flow, task\n", "from prefect.executors import DaskExecutor\n", - "from prefect.utilities.debug import raise_on_exception\n", - "from datetime import datetime as time" + "\n", + "from carbonplan_trace.tiles import tiles\n", + "from carbonplan_trace.v1 import postprocess, utils\n", + "from carbonplan_trace.v1.landsat_preprocess import access_credentials" ] }, { @@ -331,7 +317,7 @@ "\n", " for lat_increment in np.arange(0, 10, tile_degree_size):\n", " for lon_increment in np.arange(0, 10, tile_degree_size):\n", - " task_tag = \"{}_{}_{}_{}\".format(min_lat, min_lon, lat_increment, lon_increment)\n", + " task_tag = f\"{min_lat}_{min_lon}_{lat_increment}_{lon_increment}\"\n", " if task_tag in completed_subtiles:\n", " continue\n", " else:\n", @@ -450,7 +436,7 @@ " if new_state.is_running():\n", " print(\"running!\")\n", " if new_state.is_failed():\n", - " print(\"this task {} failed\".format(task))\n", + " print(f\"this task {task} failed\")\n", " raise ValueError(\"OH NO\") # function that sends a notification\n", " return new_state" ] diff --git a/notebooks/processing/prepare_allometric_equations_mask.ipynb b/notebooks/processing/prepare_allometric_equations_mask.ipynb index 993dd2c..f53a285 100644 --- a/notebooks/processing/prepare_allometric_equations_mask.ipynb +++ b/notebooks/processing/prepare_allometric_equations_mask.ipynb @@ -52,22 +52,19 @@ "metadata": {}, "outputs": [], "source": [ - "import geopandas as gpd\n", - "import pandas as pd\n", - "import numpy as np\n", - "import xarray as xr\n", - "import matplotlib.pyplot as plt\n", - "\n", - "import fsspec\n", "import os\n", "import shutil\n", - "import regionmask\n", - "import rioxarray\n", - "\n", "from itertools import product\n", - "from zarr.errors import GroupNotFoundError\n", "\n", + "import fsspec\n", + "import geopandas as gpd\n", + "import numpy as np\n", + "import pandas as pd\n", + "import regionmask\n", + "import xarray as xr\n", + "from gcsfs import GCSFileSystem\n", "from shapely.geometry import Polygon\n", + "from zarr.errors import GroupNotFoundError\n", "\n", "import carbonplan_trace.v1.utils as utils\n", "\n", @@ -76,7 +73,6 @@ "# parse_bounding_lat_lon_for_tile,\n", "# )\n", "\n", - "from gcsfs import GCSFileSystem\n", "\n", "fs = GCSFileSystem(cache_timeout=0)" ] @@ -255,7 +251,7 @@ " output_path = (\n", " f\"gs://carbonplan-climatetrace/intermediates/biomass/{lat}_{lon}.zarr/.zmetadata\"\n", " )\n", - " if not fs.exists(output_path) and not fn in no_data_tiles:\n", + " if not fs.exists(output_path) and fn not in no_data_tiles:\n", " lat_lon_tags.append((lat, lon))\n", "\n", " return lat_lon_tags\n", @@ -316,7 +312,7 @@ "outputs": [], "source": [ "# load a tile to double check output\n", - "ds = get_tile_in_xr(f\"gs://carbonplan-scratch/trace_scratch/ecoregions_mask/50N_130W.zarr\")" + "ds = get_tile_in_xr(\"gs://carbonplan-scratch/trace_scratch/ecoregions_mask/50N_130W.zarr\")" ] }, { @@ -615,7 +611,7 @@ "outputs": [], "source": [ "# load a tile to double check output\n", - "ds = get_tile_in_xr(f\"gs://carbonplan-scratch/trace_scratch/ecoregions_mask/50N_130W.zarr\")\n", + "ds = get_tile_in_xr(\"gs://carbonplan-scratch/trace_scratch/ecoregions_mask/50N_130W.zarr\")\n", "ds" ] }, @@ -789,7 +785,7 @@ "outputs": [], "source": [ "# load a tile to double check output\n", - "ds = get_tile_in_xr(f\"gs://carbonplan-scratch/trace_scratch/ecoregions_mask/70N_140W.zarr\")\n", + "ds = get_tile_in_xr(\"gs://carbonplan-scratch/trace_scratch/ecoregions_mask/70N_140W.zarr\")\n", "ds" ] }, @@ -1068,7 +1064,7 @@ "metadata": {}, "outputs": [], "source": [ - "mapper = fsspec.get_mapper(f\"gs://carbonplan-climatetrace/intermediates/igbp/50N_130W.zarr\")\n", + "mapper = fsspec.get_mapper(\"gs://carbonplan-climatetrace/intermediates/igbp/50N_130W.zarr\")\n", "test = xr.open_zarr(mapper)\n", "\n", "test.sel(year=2003).igbp[::10, ::10].plot()" @@ -1188,7 +1184,7 @@ " )\n", "\n", "final = xr.combine_by_coords(final, combine_attrs=\"drop_conflicts\")\n", - "mapper = fsspec.get_mapper(f\"gs://carbonplan-climatetrace/inputs/modis_burned_area/composite.zarr\")\n", + "mapper = fsspec.get_mapper(\"gs://carbonplan-climatetrace/inputs/modis_burned_area/composite.zarr\")\n", "final.to_zarr(mapper, mode=\"w\")" ] }, @@ -1199,7 +1195,7 @@ "metadata": {}, "outputs": [], "source": [ - "mapper = fsspec.get_mapper(f\"gs://carbonplan-climatetrace/inputs/modis_burned_area/composite.zarr\")\n", + "mapper = fsspec.get_mapper(\"gs://carbonplan-climatetrace/inputs/modis_burned_area/composite.zarr\")\n", "burned = xr.open_zarr(mapper)\n", "burned[\"burned date\"].attrs = burned.attrs" ] @@ -1303,7 +1299,7 @@ "outputs": [], "source": [ "mapper = fsspec.get_mapper(\n", - " f\"gs://carbonplan-climatetrace/intermediates/modis_burned_area/50N_130W.zarr\"\n", + " \"gs://carbonplan-climatetrace/intermediates/modis_burned_area/50N_130W.zarr\"\n", ")\n", "test = xr.open_zarr(mapper)\n", "\n", @@ -1381,10 +1377,9 @@ "metadata": {}, "outputs": [], "source": [ - "from carbonplan_trace.v1.glas_allometric_eq import (\n", - " parse_bounding_lat_lon_for_tile,\n", - ")\n", - "from shapely import geometry" + "from shapely import geometry\n", + "\n", + "from carbonplan_trace.v1.glas_allometric_eq import parse_bounding_lat_lon_for_tile" ] }, { @@ -1459,7 +1454,7 @@ "metadata": {}, "outputs": [], "source": [ - "mapper = fsspec.get_mapper(f\"gs://carbonplan-scratch/trace_scratch/ecoregions_mask/50N_080W.zarr\")\n", + "mapper = fsspec.get_mapper(\"gs://carbonplan-scratch/trace_scratch/ecoregions_mask/50N_080W.zarr\")\n", "ds = xr.open_dataset(mapper, engine=\"zarr\", cache=False)\n", "ds" ] @@ -1482,7 +1477,7 @@ "outputs": [], "source": [ "all_tiles = [\n", - " p for p in fs.ls(f\"gs://carbonplan-scratch/trace_scratch/EOSD_sorted/\") if not p.endswith(\"/\")\n", + " p for p in fs.ls(\"gs://carbonplan-scratch/trace_scratch/EOSD_sorted/\") if not p.endswith(\"/\")\n", "]\n", "\n", "for tile in all_tiles:\n", @@ -1607,7 +1602,7 @@ "metadata": {}, "outputs": [], "source": [ - "mapper = fsspec.get_mapper(f\"gs://carbonplan-scratch/trace_scratch/ecoregions_mask/70N_100W.zarr\")\n", + "mapper = fsspec.get_mapper(\"gs://carbonplan-scratch/trace_scratch/ecoregions_mask/70N_100W.zarr\")\n", "ds = xr.open_zarr(mapper)\n", "ds" ] @@ -1670,7 +1665,6 @@ "outputs": [], "source": [ "import math\n", - "import random\n", "\n", "\n", "def parse_bounding_lat_lon_for_srtm_tile(srtm_tile):\n", @@ -1835,7 +1829,7 @@ "metadata": {}, "outputs": [], "source": [ - "mapper = fsspec.get_mapper(f\"gs://carbonplan-climatetrace/intermediates/srtm/50N_130W.zarr\")\n", + "mapper = fsspec.get_mapper(\"gs://carbonplan-climatetrace/intermediates/srtm/50N_130W.zarr\")\n", "test = xr.open_zarr(mapper)\n", "test.srtm[::10, ::10].plot()" ] diff --git a/notebooks/processing/preprocess_modis_landcover_igbp.ipynb b/notebooks/processing/preprocess_modis_landcover_igbp.ipynb index ca38f23..e118130 100644 --- a/notebooks/processing/preprocess_modis_landcover_igbp.ipynb +++ b/notebooks/processing/preprocess_modis_landcover_igbp.ipynb @@ -18,24 +18,14 @@ "metadata": {}, "outputs": [], "source": [ - "import geopandas as gpd\n", - "import pandas as pd\n", - "import numpy as np\n", - "import xarray as xr\n", - "import matplotlib.pyplot as plt\n", - "\n", "import os\n", "import shutil\n", - "import regionmask\n", - "import rioxarray\n", - "\n", - "from itertools import product\n", - "from zarr.errors import GroupNotFoundError\n", "\n", - "from shapely.geometry import Polygon\n", - "\n", - "import carbonplan_trace.v1.utils as utils\n", + "import numpy as np\n", + "import pandas as pd\n", + "import xarray as xr\n", "from s3fs import S3FileSystem\n", + "from zarr.errors import GroupNotFoundError\n", "\n", "fs = S3FileSystem()" ] @@ -169,6 +159,7 @@ "source": [ "# reproject match data into 10x hansen grid, processing done for the igbp data used for allometric equation assignment\n", "\n", + "\n", "# for each tile\n", "for tp in tilepaths[103:]:\n", " print(tp)\n", @@ -230,7 +221,7 @@ "metadata": {}, "outputs": [], "source": [ - "mapper = fs.get_mapper(f\"s3://carbonplan-climatetrace/intermediate/igbp/10N_010E.zarr\")\n", + "mapper = fs.get_mapper(\"s3://carbonplan-climatetrace/intermediate/igbp/10N_010E.zarr\")\n", "test = xr.open_zarr(mapper)\n", "\n", "test.sel(year=2018).igbp[::10, ::10].plot()" @@ -245,8 +236,8 @@ "source": [ "# directly reproject the entire globe data into latlon without matching hansen grid, data used for setting global domain\n", "\n", - "local_path = f\"/home/jovyan/temp/global_igbp.zarr\"\n", - "cloud_path = f\"s3://carbonplan-climatetrace/intermediate/global_igbp.zarr\"\n", + "local_path = \"/home/jovyan/temp/global_igbp.zarr\"\n", + "cloud_path = \"s3://carbonplan-climatetrace/intermediate/global_igbp.zarr\"\n", "\n", "# for yr in np.arange(2010, 2019):\n", "# print(yr)\n", diff --git a/notebooks/processing/training_prep.ipynb b/notebooks/processing/training_prep.ipynb index fe1a2f7..d121a9f 100644 --- a/notebooks/processing/training_prep.ipynb +++ b/notebooks/processing/training_prep.ipynb @@ -18,39 +18,24 @@ "%load_ext autoreload\n", "%autoreload 2\n", "\n", - "from pyproj import CRS\n", "import boto3\n", "from rasterio.session import AWSSession\n", "from s3fs import S3FileSystem\n", "\n", "aws_session = AWSSession(boto3.Session(), requester_pays=True) # profile_name='default'),\n", "fs = S3FileSystem(requester_pays=True) # profile='default',\n", - "import xgboost as xgb\n", "\n", - "from osgeo.gdal import VSICurlClearCache\n", - "import rasterio as rio\n", - "import numpy as np\n", - "import xarray as xr\n", "import dask\n", - "import os\n", - "import fsspec\n", "import geopandas as gpd\n", - "import rioxarray # for the extension to load\n", - "import matplotlib.pyplot as plt\n", - "import utm\n", - "import pandas as pd\n", - "from datetime import datetime\n", - "import json\n", - "import zarr\n", - "import awswrangler as wr\n", + "import numpy as np\n", + "import rasterio as rio\n", "from dask_gateway import Gateway\n", - "from carbonplan_trace.v1.landsat_preprocess import access_credentials, test_credentials\n", - "from carbonplan_trace.v1.inference import predict, predict_delayed\n", + "\n", "from carbonplan_trace.v1 import utils\n", + "from carbonplan_trace.v1.landsat_preprocess import access_credentials, test_credentials\n", "from carbonplan_trace.v1.training_prep import (\n", " prep_training_dataset,\n", " prep_training_dataset_delayed,\n", - " add_parquet_urls,\n", ")" ] }, diff --git a/notebooks/processing/webmap_prep.ipynb b/notebooks/processing/webmap_prep.ipynb index eccb181..231a6a2 100644 --- a/notebooks/processing/webmap_prep.ipynb +++ b/notebooks/processing/webmap_prep.ipynb @@ -7,13 +7,11 @@ "metadata": {}, "outputs": [], "source": [ + "import numpy as np\n", "import xarray as xr\n", - "import pandas as pd\n", - "import rioxarray\n", - "from ndpyramid import pyramid_reproject\n", - "from carbonplan_data.utils import set_zarr_encoding\n", "from carbonplan_data.metadata import get_cf_global_attrs\n", - "import numpy as np" + "from carbonplan_data.utils import set_zarr_encoding\n", + "from ndpyramid import pyramid_reproject" ] }, { diff --git a/notebooks/v0/all_tiles.ipynb b/notebooks/v0/all_tiles.ipynb index 54e622c..a88b9fa 100644 --- a/notebooks/v0/all_tiles.ipynb +++ b/notebooks/v0/all_tiles.ipynb @@ -57,10 +57,10 @@ "%matplotlib inline\n", "# import dask\n", "import intake\n", - "import xarray as xr\n", - "from tqdm import tqdm\n", "import numcodecs\n", "import numpy as np\n", + "import xarray as xr\n", + "from tqdm import tqdm\n", "\n", "TC02_PER_TC = 3.67\n", "TC_PER_TBM = 0.5\n", @@ -337,7 +337,7 @@ " (da_global * da_area).coarsen(lat=100, lon=100).sum().compute()\n", " )\n", " except:\n", - " print(\"{} {} didnt work booooo\".format(lat, lon))" + " print(f\"{lat} {lon} didnt work booooo\")" ] }, { @@ -353,7 +353,7 @@ "metadata": {}, "outputs": [], "source": [ - "coarsened_url = f\"gs://carbonplan-scratch/global-forest-emissions/global/3000m/raster.zarr\"\n", + "coarsened_url = \"gs://carbonplan-scratch/global-forest-emissions/global/3000m/raster.zarr\"\n", "mapper = fsspec.get_mapper(coarsened_url)\n", "recombine = False\n", "\n", diff --git a/notebooks/v0/country_rollups.ipynb b/notebooks/v0/country_rollups.ipynb index 386f130..764fc2a 100644 --- a/notebooks/v0/country_rollups.ipynb +++ b/notebooks/v0/country_rollups.ipynb @@ -6,13 +6,13 @@ "metadata": {}, "outputs": [], "source": [ - "import fsspec\n", - "import xarray as xr\n", "import dask\n", + "import fsspec\n", "import intake\n", - "import numpy as np\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "import regionmask\n", + "import xarray as xr\n", "\n", "dask.config.set(**{\"array.slicing.split_large_chunks\": False})\n", "\n", @@ -106,7 +106,7 @@ "metadata": {}, "outputs": [], "source": [ - "coarsened_url = f\"gs://carbonplan-scratch/global-forest-emissions/americas/3000m/raster_v2.zarr\"\n", + "coarsened_url = \"gs://carbonplan-scratch/global-forest-emissions/americas/3000m/raster_v2.zarr\"\n", "mapper = fsspec.get_mapper(coarsened_url)\n", "ds_ameri = xr.open_zarr(mapper)" ] diff --git a/notebooks/v0/methods_doc_v0_figures.ipynb b/notebooks/v0/methods_doc_v0_figures.ipynb index 4affcff..afb7548 100644 --- a/notebooks/v0/methods_doc_v0_figures.ipynb +++ b/notebooks/v0/methods_doc_v0_figures.ipynb @@ -26,10 +26,9 @@ "outputs": [], "source": [ "import geopandas\n", - "import pandas as pd\n", - "from io import StringIO\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", + "import pandas as pd\n", "from carbonplan_styles.mpl import set_theme\n", "\n", "set_theme()" diff --git a/notebooks/v0/single_tile.ipynb b/notebooks/v0/single_tile.ipynb index 5c279ca..c17c7d8 100644 --- a/notebooks/v0/single_tile.ipynb +++ b/notebooks/v0/single_tile.ipynb @@ -56,14 +56,12 @@ "source": [ "%matplotlib inline\n", "import dask\n", - "import matplotlib.pyplot as plt\n", "import geopandas\n", "import intake\n", - "import xarray as xr\n", - "import rioxarray\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import seaborn as sns\n", - "from dask.distributed import Client\n", + "import xarray as xr\n", "from shapely.geometry import mapping\n", "from sklearn.metrics import mean_squared_error\n", "\n", @@ -416,7 +414,6 @@ "@dask.delayed\n", "def calc_state_biomass(source, gdf):\n", " \"\"\"lazy helper function to compute aggregated biomass by shape\"\"\"\n", - " import rioxarray\n", "\n", " # open the dataset inside the task\n", " obj = source.to_dask()[\"aboveground_biomass_carbon_2010\"] * ORNL_SCALING\n", @@ -435,6 +432,7 @@ "source": [ "# aggregate the ORNL (Spawn and Gibbs) data by state.\n", "\n", + "\n", "from dask.diagnostics import ProgressBar\n", "\n", "biomass = [calc_state_biomass(cat.ornl_biomass, us_states.iloc[[i]]) for i in range(len(us_states))]\n", diff --git a/scripts/cache_glas.py b/scripts/cache_glas.py index 90f9c1b..39a1ef2 100644 --- a/scripts/cache_glas.py +++ b/scripts/cache_glas.py @@ -29,9 +29,9 @@ URS_URL = "https://urs.earthdata.nasa.gov" CMR_PAGE_SIZE = 2000 CMR_FILE_URL = ( - "{0}/search/granules.json?provider=NSIDC_ECS" + "{}/search/granules.json?provider=NSIDC_ECS" "&sort_key[]=start_date&sort_key[]=producer_granule_id" - "&scroll=true&page_size={1}".format(CMR_URL, CMR_PAGE_SIZE) + "&scroll=true&page_size={}".format(CMR_URL, CMR_PAGE_SIZE) ) @@ -80,8 +80,8 @@ def get_credentials(url): username, account, password = info.authenticators(urlparse(URS_URL).hostname) errprefix = "netrc error: " except Exception as e: - if not ("No such file" in str(e)): - print("netrc error: {0}".format(str(e))) + if "No such file" not in str(e): + print(f"netrc error: {str(e)}") username = None password = None @@ -89,13 +89,13 @@ def get_credentials(url): if not username: username = get_username() password = get_password() - credentials = "{0}:{1}".format(username, password) + credentials = f"{username}:{password}" credentials = base64.b64encode(credentials.encode("ascii")).decode("ascii") if url: try: req = Request(url) - req.add_header("Authorization", "Basic {0}".format(credentials)) + req.add_header("Authorization", f"Basic {credentials}") opener = build_opener(HTTPCookieProcessor()) opener.open(req) except HTTPError: @@ -111,7 +111,7 @@ def get_credentials(url): def build_version_query_params(version): desired_pad_length = 3 if len(version) > desired_pad_length: - print('Version string too long: "{0}"'.format(version)) + print(f'Version string too long: "{version}"') quit() version = str(int(version)) # Strip off any leading zeros @@ -119,7 +119,7 @@ def build_version_query_params(version): while len(version) <= desired_pad_length: padded_version = version.zfill(desired_pad_length) - query_params += "&version={0}".format(padded_version) + query_params += f"&version={padded_version}" desired_pad_length -= 1 return query_params @@ -133,16 +133,16 @@ def build_cmr_query_url( polygon=None, filename_filter=None, ): - params = "&short_name={0}".format(short_name) + params = f"&short_name={short_name}" params += build_version_query_params(version) - params += "&temporal[]={0},{1}".format(time_start, time_end) + params += f"&temporal[]={time_start},{time_end}" if polygon: - params += "&polygon={0}".format(polygon) + params += f"&polygon={polygon}" elif bounding_box: - params += "&bounding_box={0}".format(bounding_box) + params += f"&bounding_box={bounding_box}" if filename_filter: option = "&options[producer_granule_id][pattern]=true" - params += "&producer_granule_id[]={0}{1}".format(filename_filter, option) + params += f"&producer_granule_id[]={filename_filter}{option}" return CMR_FILE_URL + params @@ -204,7 +204,7 @@ def cmr_search( polygon=polygon, filename_filter=filename_filter, ) - print("Querying for data:\n\t{0}\n".format(cmr_query_url)) + print(f"Querying for data:\n\t{cmr_query_url}\n") cmr_scroll_id = None ctx = ssl.create_default_context() @@ -224,7 +224,7 @@ def cmr_search( cmr_scroll_id = headers["cmr-scroll-id"] hits = int(headers["cmr-hits"]) if hits > 0: - print("Found {0} matches.".format(hits)) + print(f"Found {hits} matches.") else: print("Found no matches.") search_page = response.read() @@ -252,7 +252,7 @@ def cmr_download(urls, cache_location, credentials=None): return out url_count = len(urls) - print("Downloading {0} files...".format(url_count)) + print(f"Downloading {url_count} files...") for index, url in enumerate(urls, start=1): if not credentials and urlparse(url).scheme == "https": @@ -273,7 +273,7 @@ def cmr_download(urls, cache_location, credentials=None): # open(filename, 'wb').write(resp.content) req = Request(url) if credentials: - req.add_header("Authorization", "Basic {0}".format(credentials)) + req.add_header("Authorization", f"Basic {credentials}") opener = build_opener(HTTPCookieProcessor()) with fsspec.open(target_url, mode="wb") as target: @@ -282,11 +282,11 @@ def cmr_download(urls, cache_location, credentials=None): out.append(target_url) except HTTPError as e: - print("HTTPError {0}, {1}".format(e.code, e.reason), filename) + print(f"HTTPError {e.code}, {e.reason}", filename) except URLError as e: - print("URLError: {0}".format(e.reason), filename) - except IOError as e: - print("IOError: {0}".format(e.reason), filename) + print(f"URLError: {e.reason}", filename) + except OSError as e: + print(f"IOError: {e.reason}", filename) except KeyboardInterrupt: quit() except: @@ -295,7 +295,6 @@ def cmr_download(urls, cache_location, credentials=None): def main(): - url_list = cmr_search( short_name, version, diff --git a/scripts/preprocess_glas.py b/scripts/preprocess_glas.py index 7a00ab3..3951d8d 100644 --- a/scripts/preprocess_glas.py +++ b/scripts/preprocess_glas.py @@ -145,7 +145,6 @@ def main(products=['GLAH01', 'GLAH14']): if __name__ == '__main__': - client = Client(n_workers=12) print(client) print(client.dashboard_link)