diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 00000000..e285d975
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,21 @@
+Fixes #
+
+## Before merging into `dev`-branch, please make sure that the following points are checked:
+
+- [ ] All pre-commit tests passed locally with: `pre-commit run -a`
+- [ ] File `CHANGELOG.md` was updated
+- [ ] The docs were updated
+ - [ ] if `dataset.md`s were updated, the dataset docs were regenerated using
+ `python docs/generate_dataset_mds.py`
+
+If packages were modified:
+- [ ] File `poetry.lock` was updated with: `poetry lock`
+- [ ] A new env was successfully set up
+
+WARNING: When modifying use snakemake <=7.32.0, cf. #186
+
+If data flow was adjusted:
+- [ ] Data pipeline run finished successfully with: `snakemake -jX`
+- [ ] Esys appdata was created successfully with: `snakemake -jX make_esys_appdata`
+
+ (with `X` = desired number of cores, e.g. 1)
diff --git a/.gitignore b/.gitignore
index b29049c0..f3d4d508 100644
--- a/.gitignore
+++ b/.gitignore
@@ -71,6 +71,9 @@ instance/
# Sphinx documentation
docs/_build/
+# Mkdocs documentation
+site/
+
# PyBuilder
target/
@@ -132,3 +135,29 @@ dmypy.json
.DS_Store
.snakemake
.idea
+
+# Data files
+digipipe/store/raw/*/data/*
+!digipipe/store/raw/*/data/.gitkeep
+!digipipe/store/raw/.TEMPLATE/data/*
+
+digipipe/store/preprocessed/*/data/*
+!digipipe/store/preprocessed/*/data/.gitkeep
+!digipipe/store/preprocessed/.TEMPLATE/data/*
+
+digipipe/store/datasets/*/data/*
+!digipipe/store/datasets/*/data/.gitkeep
+!digipipe/store/datasets/.TEMPLATE/data/*
+
+digipipe/store/appdata/esys/*
+digipipe/store/appdata/*/data/*
+!digipipe/store/appdata/*/data/.gitkeep
+!digipipe/store/appdata/datapackage/data/captions/.gitkeep
+!digipipe/store/appdata/datapackage/data/geodata/.gitkeep
+!digipipe/store/appdata/datapackage/data/scalars/.gitkeep
+!digipipe/store/appdata/datapackage/data/settings/.gitkeep
+!digipipe/store/appdata/datapackage/data/sequences/.gitkeep
+!digipipe/store/appdata/*/.gitkeep
+
+digipipe/store/temp/*
+!digipipe/store/temp/.gitkeep
\ No newline at end of file
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 00000000..2c88cf98
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,107 @@
+# TODO: Find out if something needs to be changed in here in comparison to digiplan
+
+exclude: 'docs|node_modules|vendors|migrations|.git|.tox'
+default_stages: [commit]
+fail_fast: true
+
+repos:
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.3.0
+ hooks:
+ - id: check-json
+ - id: end-of-file-fixer
+ - id: trailing-whitespace
+ - id: check-added-large-files
+
+ - repo: https://github.com/charliermarsh/ruff-pre-commit
+ # Ruff version.
+ rev: 'v0.0.244'
+ hooks:
+ - id: ruff
+ args: ["--fix"]
+
+ - repo: https://github.com/pycqa/isort
+ rev: 5.12.0
+ hooks:
+ - id: isort
+ name: isort (python)
+ args: ["--line-length", "80", "black"]
+
+ - repo: local
+ hooks:
+ - id: black
+ name: black
+ entry: black
+ language: python
+ types: [python]
+ - id: flake8
+ name: flake8
+ args:
+ - --max-line-length=80
+ - --ignore=DAR101,DAR201,F821,DAR401,W503,A001,A003,E800,E722,B001,B008,B023,C408,C417,W605,B007,S101
+ entry: flake8
+ language: python
+ types: [python]
+ - id: pylint
+ name: pylint
+ entry: env DATABASE_URL=null env PROJ_LIB=null USE_DOCKER=null pylint
+ language: system
+ types: [python]
+ args:
+ [
+ "-rn", # Only display messages
+ "-sn", # Don't display the score
+ "--disable=E0602", # Disable the E0602 error
+ "--disable=C0114", # Disable the C0114 error TODO: To be fixed
+ "--disable=C0116", # Disable the C0116 error TODO: To be fixed
+ "--disable=R1729", # Disable the R1729 error TODO: To be fixed
+ "--disable=C0103", # Disable the C0103 error TODO: To be fixed
+ "--disable=R0801", # Disable the R0801 error TODO: To be fixed
+ "--disable=W1514", # Disable the W1514 error TODO: To be fixed
+ "--disable=R1734", # Disable the R1734 error TODO: To be fixed
+ "--disable=R1735", # Disable the R1735 error TODO: To be fixed
+ "--disable=W0612", # Disable the W0612 error TODO: To be fixed
+ "--disable=W1401", # Disable the W1401 error TODO: To be fixed
+ "--disable=W0511", # Disable the W0511 error TODO: To be fixed
+ "--disable=R0913", # Disable the R0913 error TODO: To be fixed
+ "--disable=R1705", # Disable the R1705 error TODO: To be fixed
+ "--disable=E0401", # Disable the E0401 error TODO: To be fixed
+ "--disable=E0611", # Disable the E0611 error TODO: To be fixed
+ "--disable=W0621", # Disable the W0621 error TODO: To be fixed
+ "--disable=W0640", # Disable the W0640 error TODO: To be fixed
+ "--disable=C0115", # Disable the C0115 error TODO: To be fixed
+ "--disable=R1725", # Disable the R1725 error TODO: To be fixed
+ "--disable=R0903", # Disable the R0903 error TODO: To be fixed
+ "--disable=C0302", # Disable the C0302 error TODO: To be fixed
+ "--disable=W0707", # Disable the W0707 error TODO: To be fixed
+ "--disable=W0108", # Disable the W0108 error TODO: To be fixed
+ "--disable=W1203", # Disable the W1203 error TODO: To be fixed
+ "--disable=W0622", # Disable the W0622 error TODO: To be fixed
+ "--disable=R1720", # Disable the R1720 error TODO: To be fixed
+ "--disable=R0914", # Disable the R0914 error TODO: To be fixed
+ "--disable=R1724", # Disable the R1724 error TODO: To be fixed
+ "--disable=W0012", # Disable the W0012 error TODO: To be fixed
+ "--disable=W0611", # Disable the W0611 error TODO: To be fixed
+ "--disable=C0207", # Disable the C0207 error TODO: To be fixed
+ "--max-line-length=80", # Set the maximum line length to 80
+ ]
+ - id: mypy
+ name: mypy
+ entry: mypy
+ language: python
+ types: [python]
+ args:
+ [
+ "--ignore-missing-imports", # Ignore missing imports
+ "--warn-unused-ignores", # Warn about unused ignore comments
+ "--disable-error-code=name-defined", # To suppress Name "snakemake"
+ # is not defined [name-defined]
+ "--disable-error-code=var-annotated",
+ "--disable-error-code=var-annotated",
+ "--disable-error-code=dict-item",
+ "--disable-error-code=arg-type",
+ "--disable-error-code=assignment",
+ "--disable-error-code=attr-defined",
+ "--disable-error-code=index",
+ "--disable-error-code=misc",
+ ]
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 00000000..ba84eb89
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,23 @@
+# Read the Docs configuration file for MkDocs projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+ os: ubuntu-22.04
+ tools:
+ python: "3.11"
+ jobs:
+ pre_install:
+ - python docs/generate_dataset_mds.py
+
+
+mkdocs:
+ configuration: mkdocs.yml
+
+# Optionally declare the Python requirements required to build your docs
+python:
+ install:
+ - requirements: docs/requirements.txt
\ No newline at end of file
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 52377f3e..bc02224c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,12 +4,155 @@ All notable changes to this project will be documented in this file.
The format is inspired from [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
and the versioning aim to respect [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
-## [Unreleased]
+## [1.0.0] - 2023-11-07
### Added
+
- Setup initial structure and files
+- Add first bunch of datasets for testing the data flow
+- Modularize datasets
+- Add first draft of RTD docs
+- Add dataset: BNetzA MaStR
+- Add Nominatim geocoder
+- Add dataset: population
+- Clean rule
+- Datasets attribute captions
+- Create list of region-specific datasets in the docs
+- pyproject.toml and poetry.lock file have been added with the conversion to poetry
+- Add pre-commit in order to check for errors and linting bugs before commits
+- Add types-pyyaml package
+- Add dataset: employees and companies
+- Add dataset: demandregio electricity demand
+- Add dataset: BMWK long-term scenarios
+- Add rules to download raw data (zipped) from cloud, extract and copy to 'store/raw'
+- Add module 'data_io', containing relevant helper functions (downloading, extracting, copying, cleaning)
+- Create metadata
+- Add script to generate dataset md files for documentation
+- Add dataset: demand_heat_region
+- Add dataset: heatpump_cop
+- Add dataset: stala_st_energy
+- Add dataset: eurostat_lau
+- Add dataset: regstat_energy
+- Add dataset: dwd_temperature
+- Add dataset: ageb_energy_balance
+- Add dataset: seenergies_peta5
+- Add dataset: renewables.ninja_feedin
+- Add dataset: renewable_feedin
+- Add dataset: bnetza_mastr_correction_region and correct wrong units
+- Integrate building of energy system for appdata in pipeline via dir *esys*
+- Update store with dir structure for *esys* data
+- Add creation of empty time series for the *esys*
+- Add dataset: rpg_abw_regional_plan
+- Add dataset: potentialarea_wind_region
+- Add writing of default values to *esys* raw scalar data
+- Add datasets: rli_pv_wfr and rli_pv_wfr_region
+- Add module appdata to workflow
+- Add dataset: geodata_infolayers
+- Add dataset: potentialarea_pv_ground_region
+- Add dataset: app datapackage
+- Add dataset: potentialarea_pv_roof_region
+- Add dataset: technology_data
+- Add dataset: settings
+- Calc panel settings from datasets
+- Add dataset: osm (Germany)
+- Add dataset: osm_buildings and add stats on ground areas
+- Add mapping of costs and efficiencies from store/raw to store/datasets
+- Add dataset: emissions
+- Add captions to app datapackage (here: MaStR, heat, potentialarea_wind)
+- Add mapping of time series data in datasets to empty time series according to
+ the mapping provided in map_ts.yml
+- Add build configuration for readthedocs
+- Add creation of stats of development over time for bnetza_mastr_wind_region,
+ bnetza_mastr_pv_ground_region, bnetza_mastr_pv_roof_region
+- Add dataset: dbfz_biomass_heat_capacities
+- Add the calculation of relative demand of biomass conversion technologies via
+ their relative capacities
+- Add deletion of all data in store/datasets/esys_raw/data
+- Add notes on OSM download and run resources
+- Add nominal power per wind turbine for 2045
+- Add technology data for batteries
+- Add technology data for thermal storages
+- Add definition of relevant regions (NUTS3) in global configuration file
+- Add documentation of energy system of ABW
### Changed
+- Move dataset docs from config to md files
+- Retain mastr_id in MaStR datasets
+- Fix loading of empty yml files
+- Fix loading of global config when in workflow dir
+- Integrate esys Snakefile in workflow Snakefile and update clean rule
+- Fix shapely deprecation warning
+- Fix ogr2ogr conversion with recent GDAL version (v3.6.2)
+- Fix conda installation by removing python-gdal from environment.yml
+- The package management in digipipe has been changed to poetry.
+- The installation of a virtual environment is done only from the environment.yml file and via conda.
+- Apply linters on repo among others: black, isort, check-json and end-of-file-fixer
+- Update population with prognoses from demandregio dataset
+- Fix C419 flake8 error
+- Switch to mkdocs for documentation (Sphinx deleted)
+- Normalize renewable feedin timeseries
+- Fix instruction to obtain raw files
+- Translate all dataset.md files to German
+- Exchange *Test_scenario* with *2045_scenario* in *digipipe/esys/scenarios*
+- Split each demand per sector in *esys*
+- File .gitignore again includes ignoring of esys appdata
+- pv_roof area stats: distinguish between all and non-historic buildings
+- storage units: add region-wide values for spec. capacity and power for those
+ connected to PV roof units
+- Add data on installed el. power to bmwk_long_term_scenarios
+- Disaggregate PV state targets to region in potentialarea_pv_ground_region
+- Adapt osm_filtered to ose osm dataset and extract building data
+- Disaggregate PV state targets to region in potentialarea_pv_roof_region
+- Changes were applied to the energy system. Among others RoR, small batteries
+ and biogas were added. A distinction was made between centralized and
+ decentralized CHPs
+- Scenario 2045_scenario needs default_scalars.csv instead of scalars.csv
+- By default set costs and efficiencies of esys are written to
+ default_scalars.csv instead of default_costs_efficiencies.csv
+- Default variable_costs are passed with input_parameters for storages
+- Pass time series instead of scalar with efficiency for central heat pump
+- Fix wind+pv_ground default values in panel settings
+- Set all default control values in panel settings
+- Kick biogas shortage
+- Rename dataset captions to app_captions
+- Move app settings to datasets and include in app datapackage
+- Adapt 2045_scenario.yml so that time series with values are used instead of
+ empty ts
+- Suppress warning of loosing data in source and comment columns while
+ unstacking if they are empty
+- Change max. installable PV roof capacity in panel settings
+- Fix panel settings for large batteries
+- Add additional captions to MaStR captions
+- Use LTS version of OSM
+- The unstacking of time series in esys was fixed so that warning is given if
+ there is at least one value in columns 'source' or 'comment'
+- Minor fix applied reformatting with black
+- Only use operating units from mastr for municipality stats and temporal
+ development
+- Heat pump ASHP/GSHP split fixed
+- Replace the relative demand of biomass with the relative demand of each
+ biomass conversion technology
+- Fix clean rule
+- Update raw datapackage URL
+- Restrict snakemake version to v7.32.0
+- Add central heat pump targets to slider
+- Restrict heat pump sliders to not move under 50%
+- Fix pv ground slider values to prevent app to alter SQ value from panel
+ settings
+- Fix PV roof slider values
+- Add HP share slider from-max values to prevent 100 % HP share
+- Updated technology_data dataset.md and metadata
+- Update metadate in store_raw
+- Adapt existing dataset rules to use the global region definition (NUTS3)
+- Update list of region-specific datasets
+
### Removed
+- setup.py and requirements.txt files are omitted with the conversion to poetry
+- sphinx from poetry environment
+- Remove dataset: osm_forest
+- Obsolete targets from rule all
+- Merge dataset costs_efficiencies into technology_data
+- Merge dataset costs_efficiencies into technology_data
+- Remove values for redundant subsliders from app datapackage
diff --git a/PIPELINE.md b/PIPELINE.md
deleted file mode 100644
index 45d269a0..00000000
--- a/PIPELINE.md
+++ /dev/null
@@ -1 +0,0 @@
-# Pipeline docs
diff --git a/README.md b/README.md
index 3183b008..37bb9966 100644
--- a/README.md
+++ b/README.md
@@ -1,51 +1,17 @@
-
+
+--.
+
+ Parameters
+ ----------
+ region : str
+ region
+ carrier : str
+ carrier
+ tech : str
+ technology
+
+ Returns
+ -------
+ String containing name according to convention eg. B-ch4-gt.
+
+ """
+ return f"{region}-{carrier}-{tech}"
+
+ def get_name_for_df(df):
+ r"""
+ This function returns a series of names generated from the convention.
+
+ Parameters
+ ----------
+ df : pd.DataFrame
+ DataFrame in oemof-B3-resources format.
+
+ Returns
+ -------
+ pd.Series with names set according to convention in get_name.
+
+ """
+ # Check if carrier, region and tech exist as columns
+ if {"carrier", "region", "tech"}.issubset(df.columns):
+ return df.apply(
+ lambda x: get_name(x["region"], x["carrier"], x["tech"]), 1
+ )
+ else:
+ raise KeyError(
+ "Please provide a DataFrame that conforms to "
+ "oemof-B3-resources-format."
+ )
+
+ def check_name(df):
+ r"""
+ This function checks whether a name given by the user matches the one
+ from the oemof-B3 naming convention. It prints a warning if expected
+ differs from given names and prints a list with expected names.
+
+ Parameters
+ ----------
+ df : pd.DataFrame
+ DataFrame in oemof-B3-resources format.
+
+ """
+ # Get name of Dataframe
+ name_as_given = df["name"]
+
+ # Get name according to convention
+ name_generated = get_name_for_df(df)
+
+ # Get diff of names
+ diff_in_name = compare_scalar_data(name_as_given, name_generated)
+
+ # Save unique values of diff to a list and print as warning
+ expected_names = list(diff_in_name.unique())
+ diff_names = list(
+ compare_scalar_data(name_generated, name_as_given).unique()
+ )
+ if expected_names:
+ logger.warning(
+ "The name you have set for some of your scalar data differs "
+ "from the convention (--). \n"
+ "We expected but could not find the following name(s): "
+ f"{expected_names}."
+ "\nInstead we got the name(s): "
+ f"{diff_names}"
+ )
+ if overwrite:
+ logger.warning(
+ "The names will be overwritten with names following the"
+ "convention"
+ )
+
+ def set_name(df, overwrite):
+ r"""
+ This function
+ 1. checks the name if the name is not empty and
+ 2. overwrites the name with the generated name if overwrite is true
+ or the name is empty
+
+ Parameters
+ ----------
+ df : pd.DataFrame
+ DataFrame in oemof-B3-resources format.
+
+ Returns
+ -------
+ df : pd.DataFrame
+ DataFrame in oemof-B3-resources format and formatted name.
+
+ """
+ all_empty = df["name"].isnull().values.all()
+ if not all_empty:
+ check_name(df)
+
+ elif all_empty or overwrite:
+ name_generated = get_name_for_df(df)
+ _df = df.copy() # To avoid SettingWithCopyWarning
+ _df.loc[:, "name"] = name_generated
+ df = _df
+
+ return df
+
+ def compare_scalar_data(sc_1, sc_2):
+ r"""
+ This functions compares the column of two DataFrames
+ It returns a DataFrame with scalars that diverge in name convention.
+
+ Parameters
+ ----------
+ sc_1 : pd.Series
+ Series with given values
+ sc_2 : pd.Series
+ Series with expected values
+
+ Returns
+ -------
+ diff_name_sc : pd.Series
+ Series where expected values not found
+ """
+ diff_name_sc = sc_1.compare(sc_2)
+
+ return diff_name_sc["other"]
+
+ # PART 1: Ensure name is set (according to convention) where name is empty
+ # and region is fixed
+ # Save values where name is None and region is not "ALL" in new DataFrame
+ sc_wo_name = sc[sc["name"].isnull()]
+ sc_add_name = set_name(sc_wo_name, overwrite)
+
+ # PART 2: Ensure name is checked for all values that are not None and where
+ # region is fixed
+ sc_with_name = sc[sc["name"].notnull()]
+ sc_with_name = set_name(sc_with_name, overwrite)
+
+ # PART 3: Concatenate DataFrame with corrected name and DataFrame with set
+ # name
+ scalars_set_name = pd.concat([sc_with_name, sc_add_name])
+
+ return scalars_set_name
+
+
+def expand_regions(scalars, regions, where="ALL"):
+ r"""
+ Expects scalars in oemof_b3 format (defined in
+ ''oemof_b3/schema/scalars.csv'') and regions.
+ Returns scalars with new rows included for each region in those places
+ where region equals
+ `where`.
+
+ Parameters
+ ----------
+ scalars : pd.DataFrame
+ Data in oemof_b3 format to expand
+ regions : list
+ List of regions
+ where : str
+ Key that should be expanded
+ Returns
+ -------
+ sc_with_region : pd.DataFrame
+ Data with expanded regions in oemof_b3 format
+ """
+ _scalars = format_header(
+ scalars, HEADER_B3_SCAL, esys_conf.settings.general.scal_index_name
+ )
+
+ sc_with_region = _scalars.loc[scalars["region"] != where, :].copy()
+
+ sc_wo_region = _scalars.loc[scalars["region"] == where, :].copy()
+
+ # REGIONALIZATION
+ if not sc_wo_region.empty:
+ # Ensure name is empty if region is 'ALL'
+ # Print user warning if name is not NaN and region is "ALL"
+ name_overwritten = sc_wo_region[sc_wo_region["name"].notnull()][
+ "name"
+ ].values
+ if not sc_wo_region["name"].isnull().values.all():
+ print(
+ "User warning: Please leave 'name' empty if you set 'region' "
+ "to 'ALL'.\n"
+ "The name you have specified "
+ f"{name_overwritten} "
+ f"will be overwritten."
+ )
+
+ # Set region
+ for region in regions:
+ regionalized = sc_wo_region.copy()
+ regionalized["region"] = region
+
+ sc_with_region = pd.concat([sc_with_region, regionalized])
+
+ sc_with_region = sc_with_region.reset_index(drop=True)
+
+ sc_with_region.index.name = esys_conf.settings.general.scal_index_name
+
+ return sc_with_region
+
+
+def merge_a_into_b(df_a, df_b, on, how="left", indicator=False, verbose=True):
+ r"""
+ Writes scalar data from df_a into df_b, according to 'on'. Where df_a
+ provides no data, the values of df_b are used. If how='outer', data from
+ df_a that is not in df_b will be kept.
+
+ Parameters
+ ----------
+ df_a : pd.DataFrame
+ DataFrame in oemof_b3 scalars format
+ df_b : pd.DataFrame
+ DataFrame in oemof_b3 scalars format
+ on : list
+ List of columns to merge on
+ how : str
+ 'left' or 'outer'. Default: 'left'
+ indicator : bool
+ If True, an indicator column is included. Default: False
+
+ Returns
+ -------
+ merged : pd.DataFrame
+ DataFrame in oemof_b3 scalars format.
+ """
+ _df_a = df_a.copy()
+ _df_b = df_b.copy()
+
+ # save df_b's index name and column order
+ df_b_index_name = _df_b.index.name
+ df_b_columns = list(_df_b.columns)
+ if indicator:
+ df_b_columns.append("_merge")
+
+ # Give some information on how the merge affects the data
+ set_index_a = set(
+ map(tuple, pd.Index(_df_a.loc[:, on].replace(np.nan, "NaN")))
+ )
+ set_index_b = set(
+ map(tuple, pd.Index(_df_b.loc[:, on].replace(np.nan, "NaN")))
+ )
+
+ if verbose:
+ a_not_b = set_index_a.difference(set_index_b)
+ if a_not_b:
+ if how == "left":
+ logger.warning(
+ f"There are {len(a_not_b)} elements in df_a but not in "
+ "df_b and are lost (choose how='outer' to keep them): "
+ f"{a_not_b}"
+ )
+ elif how == "outer":
+ logger.info(
+ f"There are {len(a_not_b)} elements in df_a that are"
+ f" added to df_b: {a_not_b}"
+ )
+
+ a_and_b = set_index_a.intersection(set_index_b)
+ logger.info(
+ f"There are {len(a_and_b)} elements in df_b that are updated by "
+ "df_a."
+ )
+
+ b_not_a = set_index_b.difference(set_index_a)
+ logger.info(
+ f"There are {len(b_not_a)} elements in df_b that are unchanged: "
+ f"{b_not_a}"
+ )
+
+ # Merge a with b, ignoring all data in b
+ merged = _df_b.drop(columns=_df_b.columns.drop(on)).merge(
+ _df_a,
+ on=on,
+ how=how,
+ indicator=indicator,
+ sort=False,
+ )
+
+ merged.index.name = df_b_index_name
+
+ # Where df_a contains no data, use df_b
+ merged = merged.reset_index().set_index(
+ on
+ ) # First reset, then set index to keep it as a column
+
+ merged.update(_df_b.set_index(on), overwrite=False)
+
+ # Set original index and recover column order
+ merged = merged.reset_index().set_index(df_b_index_name)
+
+ merged = merged[df_b_columns]
+
+ return merged
+
+
+def check_consistency_timeindex(df, index):
+ """
+ This function assert that values of a column in a stacked DataFrame are
+ same for all time steps.
+
+ Parameters
+ ----------
+ df : pandas.DataFrame
+ DataFrame for which the time index is checked
+ index : string
+ Index of values to be checked in the DataFrame
+
+ Returns
+ -------
+ value : string
+ Single value of the series of duplicates
+ """
+ if index == "timeindex_start":
+ name = "start date"
+ elif index == "timeindex_stop":
+ name = "end date"
+ elif index == "timeindex_resolution":
+ name = "frequency"
+
+ if np.all(df[index].array == df[index].array[0]):
+ value = df[index].array[0]
+ if value is None:
+ raise TypeError(
+ f"Your provided data is missing a {name}."
+ f"Please make sure you pass the {name} with {index}."
+ )
+ else:
+ return value
+ else:
+ raise ValueError(
+ f"The {name} of your provided data doesn't match for all entries. "
+ f"Please make sure to pass the {name} with {index}."
+ )
+
+
+def stack_timeseries(df):
+ """
+ This function stacks a Dataframe in a form where one series resides in one
+ row.
+
+ Parameters
+ ----------
+ df : pandas.DataFrame
+ DataFrame to be stacked
+
+ Returns
+ -------
+ df_stacked : pandas.DataFrame
+ Stacked DataFrame
+ """
+ _df = df.copy()
+
+ # Assert that _df has a timeindex
+ if not isinstance(_df.index, pd.DatetimeIndex):
+ raise TypeError(
+ "Your data should have a time series as an index of the format "
+ "'%Y-%m-%d %H:%M:%S'."
+ )
+
+ # Assert that frequency match for all time steps
+ if pd.infer_freq(_df.index) is None:
+ raise TypeError(
+ "No frequency of your provided data could be detected."
+ "Please provide a DataFrame with a specific frequency (eg. 'H' or "
+ "'T')."
+ )
+
+ _df_freq = pd.infer_freq(_df.index)
+ if _df.index.freqstr is None:
+ logger.info(
+ f"The frequency of your data is not specified in the DataFrame, "
+ f"but is of the following frequency alias: {_df_freq}. "
+ f"The frequency of your DataFrame is therefore automatically set "
+ f"to the frequency with this alias."
+ )
+ _df = _df.asfreq(_df_freq)
+
+ # Stack timeseries
+ df_stacked_cols = [
+ "var_name",
+ "timeindex_start",
+ "timeindex_stop",
+ "timeindex_resolution",
+ "series",
+ ]
+
+ df_stacked = pd.DataFrame(columns=df_stacked_cols)
+
+ timeindex_start = _df.index.values[0]
+ timeindex_stop = _df.index.values[-1]
+
+ for column in df.columns:
+ var_name = column
+ timeindex_resolution = _df[column].index.freqstr
+ series = [list(_df[column].values)]
+
+ column_data = [
+ var_name,
+ timeindex_start,
+ timeindex_stop,
+ timeindex_resolution,
+ series,
+ ]
+
+ dict_stacked_column = dict(zip(df_stacked_cols, column_data))
+ df_stacked_column = pd.DataFrame(data=dict_stacked_column)
+ df_stacked = pd.concat(
+ [df_stacked, df_stacked_column], ignore_index=True
+ )
+
+ # Save name of the index in the unstacked DataFrame as name of the index of
+ # "timeindex_start" column of stacked DataFrame, so that it can be
+ # extracted from it when unstacked again.
+ df_stacked["timeindex_start"].index.name = _df.index.name
+
+ return df_stacked
+
+
+def unstack_timeseries(df):
+ """
+ This function unstacks a Dataframe so that there is a row for each value.
+
+ Parameters
+ ----------
+ df : pandas.DataFrame
+ DataFrame to be unstacked
+
+ Returns
+ -------
+ df_unstacked : pandas.DataFrame
+ Unstacked DataFrame
+ """
+ _df = df.copy()
+
+ # Assert that frequency match for all time steps
+ frequency = check_consistency_timeindex(_df, "timeindex_resolution")
+ timeindex_start = check_consistency_timeindex(_df, "timeindex_start")
+ timeindex_stop = check_consistency_timeindex(_df, "timeindex_stop")
+
+ # Warn user if "source" or "comment" in columns of stacked DataFrame
+ # These two columns will be lost once unstacked
+ lost_columns = ["source", "comment"]
+ for col in lost_columns:
+ if col in list(df.columns):
+ if not _df[col].isna().all() or _df[col].values.all() == "None":
+ logger.warning(
+ f"Caution any remarks in column '{col}' are lost after "
+ f"unstacking."
+ )
+
+ # Process values of series
+ values_series = []
+ for row in _df.iterrows():
+ values_series.append(row[1]["series"])
+
+ values_array = np.array(values_series).transpose()
+
+ # Unstack timeseries
+ df_unstacked = pd.DataFrame(
+ values_array,
+ columns=list(_df["var_name"]),
+ index=pd.date_range(timeindex_start, timeindex_stop, freq=frequency),
+ )
+
+ # Get and set index name from and to index name of "timeindex_start".
+ # If it existed in the origin DataFrame, which has been stacked, it will be
+ # set to that one.
+ df_unstacked.index.name = _df["timeindex_start"].index.name
+
+ return df_unstacked
+
+
+def unstack_var_name(df):
+ r"""
+ Given a DataFrame in oemof_b3 scalars format, this function will unstack
+ the variables. The returned DataFrame will have one column for each
+ var_name.
+
+ Parameters
+ ----------
+ df : pd.DataFrame
+ Stacked scalar data.
+ Returns
+ -------
+ unstacked : pd.DataFrame
+ Unstacked scalar data.
+ """
+ _df = df.copy()
+
+ _df = format_header(
+ _df, HEADER_B3_SCAL, esys_conf.settings.general.scal_index_name
+ )
+
+ _df = _df.set_index(
+ [
+ "scenario_key",
+ "name",
+ "region",
+ "carrier",
+ "tech",
+ "type",
+ "var_name",
+ ]
+ )
+
+ unstacked = _df.unstack("var_name")
+
+ new_index = _df.index.droplevel(-1).unique()
+ unstacked = unstacked.reindex(new_index)
+
+ return unstacked
+
+
+def stack_var_name(df):
+ r"""
+ Given a DataFrame, this function will stack the variables and format
+ the results in b3-format.
+
+ Parameters
+ ----------
+ df : pd.DataFrame
+ DataFrame with one column per variable
+
+ Returns
+ -------
+ stacked : pd.DataFrame
+ DataFrame with a column "var_name" and "var_value"
+ """
+ assert isinstance(df, pd.DataFrame)
+
+ _df = df.copy()
+
+ _df.columns.name = "var_name"
+
+ stacked = _df.stack("var_name")
+
+ stacked.name = "var_value"
+
+ stacked = pd.DataFrame(stacked).reset_index()
+
+ stacked = sort_values(stacked)
+
+ stacked = format_header(
+ stacked, HEADER_B3_SCAL, esys_conf.settings.general.scal_index_name
+ )
+
+ return stacked
+
+
+def round_setting_int(df, decimals):
+ r"""
+ Rounds the columns of a DataFrame to the specified decimals. For zero
+ decimals, it changes the dtype to Int64. Tolerates NaNs.
+ """
+ _df = df.copy()
+
+ for col, dec in decimals.items():
+ if col not in _df.columns:
+ logger.warning(
+ f"No column named '{col}' found when trying to round."
+ )
+ continue
+ elif dec == 0:
+ dtype = "Int64"
+ else:
+ dtype = float
+
+ _df[col] = (
+ pd.to_numeric(_df[col], errors="coerce").round(dec).astype(dtype)
+ )
+
+ return _df
+
+
+def prepare_b3_timeseries(df_year, **kwargs):
+ """
+ This function takes time series in column format, stacks them, assigns
+ values to additional columns and formats the header in order to prepare
+ data in a b3 time series format
+
+ Parameters
+ ----------
+ df_year : pd.Dataframe
+ DataFrame with total normalized data in year to be processed
+ kwargs : Additional keyword arguments
+ time series data (region, scenario key and unit)
+
+ Returns
+ -------
+ df_stacked : pd.DataFrame
+ DataFrame that contains stacked time series
+
+ """
+ # Stack time series with data of a year
+ df_year_stacked = stack_timeseries(df_year)
+
+ # Add region, scenario key and unit to stacked time series
+ for key, value in kwargs.items():
+ df_year_stacked[key] = value
+
+ # Make sure that header is in correct format
+ df_year_stacked = format_header(
+ df_year_stacked, HEADER_B3_TS, esys_conf.settings.general.ts_index_name
+ )
+
+ return df_year_stacked
+
+
+def _get_component_id_in_tuple(oemof_tuple, delimiter="-"):
+ r"""
+ Returns the id of the component in an oemof tuple.
+ If the component is first in the tuple, will return 0,
+ if it is second, 1.
+
+ Parameters
+ ----------
+ oemof_tuple : tuple
+ tuple of the form (node, node) or (node, None).
+
+ Returns
+ -------
+ component_id : int
+ Position of the component in the tuple
+ """
+ # TODO: This is a dummy implementation that can easily fail
+ logger.warning(
+ "The implementation of _get_component_id_in_tuple is perliminary and "
+ "not very robust."
+ )
+ return max(
+ enumerate(oemof_tuple), key=lambda x: len(x[1].split(delimiter))
+ )[0]
+
+
+def _get_component_from_tuple(_tuple, delimiter="-"):
+ # TODO: This is a dummy implementation that can easily fail
+ logger.warning(
+ "The implementation of _get_component_from_tuple is perliminary and "
+ "not very robust."
+ )
+ return max(_tuple, key=lambda x: len(x.split(delimiter)))
+
+
+def _get_direction(oemof_tuple):
+ comp_id = _get_component_id_in_tuple(oemof_tuple)
+
+ directions = {
+ 0: "out",
+ 1: "in",
+ }
+
+ other_id = {
+ 0: 1,
+ 1: 0,
+ }[comp_id]
+
+ if oemof_tuple[other_id] == "nan":
+ return ""
+ else:
+ return directions[comp_id]
+
+
+def _get_region_carrier_tech_from_component(component, delimiter="-"):
+
+ if isinstance(component, oemof.tabular.facades.Facade):
+ region = component.region
+ carrier = component.carrier
+ tech = component.tech
+
+ elif isinstance(component, str):
+ split = component.split(delimiter)
+
+ if len(split) == 3:
+ region, carrier, tech = split
+
+ if len(split) > 3:
+
+ region, carrier, tech = "-".join(split[:2]), *split[2:]
+ warnings.warn(
+ f"Could not get region, carrier and tech by splitting "
+ f"component name into {split}. Assumed region='{region}', "
+ f"carrier='{carrier}', tech='{tech}'"
+ )
+
+ return region, carrier, tech
+
+
+def oemof_results_ts_to_oemof_b3(df):
+ r"""
+ Transforms data in oemof-tabular/oemoflex format to stacked b3 timeseries
+ format.
+
+ Parameters
+ ----------
+ df : pd.DataFrame
+ Time series in oemof-tabular/oemoflex format.
+
+ Returns
+ -------
+ df : pd.DataFrame
+ Time series in oemof-tabular/oemoflex format.
+ """
+ _df = df.copy()
+
+ # The columns of oemof results are multiindex with 3 levels: (from, to,
+ # type). This is mapped to var_name = _ with "in" if bus
+ # comes first (from), "out" if bus is second (to). If the multiindex entry
+ # is of the form (component, None, type), then var_name = type
+ component = df.columns.droplevel(2).map(_get_component_from_tuple)
+
+ # specify direction in var_name
+ direction = df.columns.droplevel(2).map(_get_direction)
+
+ var_name = df.columns.get_level_values(2)
+
+ var_name = list(zip(var_name, direction))
+
+ var_name = list(map(lambda x: "_".join(filter(None, x)), var_name))
+
+ # Introduce arbitrary unique columns before stacking.
+ _df.columns = range(len(_df.columns))
+
+ _df = stack_timeseries(_df)
+
+ # assign values to other columns
+ _df["region"], _df["carrier"], _df["tech"] = zip(
+ *component.map(_get_region_carrier_tech_from_component)
+ )
+
+ _df["name"] = component
+
+ _df["var_name"] = var_name
+
+ # ensure that the format follows b3 schema
+ _df = format_header(_df, HEADER_B3_TS, "id_ts")
+
+ return _df
+
+
+class ScalarProcessor:
+ r"""
+ This class allows to filter and unstack scalar data in a way that makes
+ processing simpler.
+ """
+
+ def __init__(self, scalars):
+ self.scalars = scalars
+
+ def get_unstacked_var(self, var_name):
+ r"""
+ Filters the scalars for the given var_name and returns the data in
+ unstacked form.
+
+ Parameters
+ ----------
+ var_name : str
+ Name of the variable
+
+ Returns
+ -------
+ result : pd.DataFrame
+ Data in unstacked form.
+ """
+ _df = filter_df(self.scalars, "var_name", var_name)
+
+ if _df.empty:
+ raise ValueError(f"No entries for {var_name} in df.")
+
+ _df = unstack_var_name(_df)
+
+ result = _df.loc[:, "var_value"]
+
+ return result
+
+ def drop(self, var_name):
+
+ self.scalars = filter_df(
+ self.scalars, "var_name", var_name, inverse=True
+ )
+
+ def append(self, var_name, data):
+ r"""
+ Accepts a Series or DataFrame in unstacked form and appends it to the
+ scalars.
+
+ Parameters
+ ----------
+ var_name : str
+ Name of the data to append
+ data : pd.Series or pd.DataFrame
+ Data to append
+
+ Returns
+ -------
+ None
+ """
+ assert not data.isna().all(), "Cannot append all NaN data."
+
+ _df = data.copy()
+
+ if isinstance(_df, pd.Series):
+ _df.name = var_name
+
+ _df = pd.DataFrame(_df)
+
+ _df = stack_var_name(_df)
+
+ self.scalars = pd.concat([self.scalars, _df])
diff --git a/digipipe/esys/esys/tools/equate_flows.py b/digipipe/esys/esys/tools/equate_flows.py
new file mode 100644
index 00000000..53bc1040
--- /dev/null
+++ b/digipipe/esys/esys/tools/equate_flows.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+
+"""Constraints to relate variables in an existing model.
+
+This script is copied from oemof.solph
+https://github.com/oemof/oemof-solph/blob/features/equate-flows/src/oemof/solph
+/constraints/equate_flows.py
+and can be deleted once oemof.tabular is updated to the new oemof version and
+oemof-B3 installs oemof.solph version (or branch) containing this script.
+
+SPDX-FileCopyrightText: Uwe Krien
+SPDX-FileCopyrightText: Simon Hilpert
+
+SPDX-License-Identifier: MIT
+
+"""
+
+from pyomo import environ as po
+
+
+def equate_flows(model, flows1, flows2, factor1=1, name="equate_flows"):
+ r"""
+ Adds a constraint to the given model that sets the sum of two groups of
+ flows equal or proportional by a factor.
+ """
+
+ def _equate_flow_groups_rule(m):
+ for ts in m.TIMESTEPS:
+ sum1_t = sum(m.flow[fi, fo, ts] for fi, fo in flows1)
+ sum2_t = sum(m.flow[fi, fo, ts] for fi, fo in flows2)
+ expr = sum1_t * factor1 == sum2_t
+ if expr is not True:
+ getattr(m, name).add(ts, expr)
+
+ setattr(
+ model,
+ name,
+ po.Constraint(model.TIMESTEPS, noruleinit=True),
+ )
+ setattr(
+ model,
+ name + "_build",
+ po.BuildAction(rule=_equate_flow_groups_rule),
+ )
+
+ return model
+
+
+def equate_flows_by_keyword(
+ model, keyword1, keyword2, factor1=1, name="equate_flows"
+):
+ r"""
+ This wrapper for equate_flows allows to equate groups of flows by using a
+ keyword instead of a list of flows.
+ """
+ flows = {}
+ for n, keyword in enumerate([keyword1, keyword2]):
+ flows[n] = []
+ for (i, o) in model.flows:
+ if hasattr(model.flows[i, o], keyword):
+ flows[n].append((i, o))
+
+ return equate_flows(model, flows[0], flows[1], factor1=factor1, name=name)
diff --git a/digipipe/esys/esys/tools/timing.py b/digipipe/esys/esys/tools/timing.py
new file mode 100644
index 00000000..5aeae166
--- /dev/null
+++ b/digipipe/esys/esys/tools/timing.py
@@ -0,0 +1,45 @@
+r"""
+A Timer class, adapted from https://github.com/realpython/codetiming
+"""
+import datetime
+import time
+
+
+class Timer:
+ """
+ A timer utility to measure the elapsed time of code blocks.
+ adapted from https://github.com/realpython/codetiming
+
+ Usage:
+ - Create a new timer with a descriptive text message.
+ - Start the timer with the `start()` method.
+ - Stop the timer with the `stop()` method to report the elapsed time.
+ """
+
+ def __init__(self, text, logger=print):
+ self._start_time = None
+ self.text = text
+ self.logger = logger
+
+ def start(self):
+ """Start a new timer"""
+ self._start_time = time.perf_counter()
+
+ def stop(self):
+ """Stop the timer, and report the elapsed time"""
+ elapsed_time = time.perf_counter() - self._start_time
+ self._start_time = None
+ if self.logger:
+ self.logger(
+ self.text
+ + f" Elapsed time: {datetime.timedelta(seconds=elapsed_time)}"
+ )
+
+ def __enter__(self):
+ """Start a new timer as a context manager"""
+ self.start()
+ return self
+
+ def __exit__(self, *exc_info):
+ """Stop the context manager timer"""
+ self.stop()
diff --git a/digipipe/esys/scenarios/2045_scenario.yml b/digipipe/esys/scenarios/2045_scenario.yml
new file mode 100644
index 00000000..a2ab26a6
--- /dev/null
+++ b/digipipe/esys/scenarios/2045_scenario.yml
@@ -0,0 +1,45 @@
+# This scenario config file contains information for several steps in the model
+# pipeline, i.e. build_datapackage, parametrize, optimize and potentially also
+# postprocess or plot.
+
+# build_datapackage
+
+name: 2045_scenario
+label: 2045-Scenario
+
+datetimeindex:
+ start:
+ "2019-01-01 00:00:00"
+ freq:
+ "H"
+ periods:
+ 8760
+
+model_structure:
+ model_structure_full
+
+# parametrize
+
+paths_scalars:
+ - store/datasets/esys_raw/data/scalars/default_scalars.csv
+ - store/appdata/esys/_resources/scal_costs_efficiencies.csv
+
+filter_scalars:
+ 1:
+ scenario_key:
+ - "ALL"
+ - "2045_scenario"
+
+paths_resources:
+
+
+paths_timeseries:
+ - store/datasets/esys_raw/data/time_series/ts_efficiencies.csv
+ - store/datasets/esys_raw/data/time_series/ts_feedin.csv
+ - store/datasets/esys_raw/data/time_series/ts_load.csv
+
+filter_timeseries:
+ scenario_key:
+ - "empty_ts"
+ timeindex_start:
+ "2019-01-01 00:00:00"
diff --git a/digipipe/esys/scripts/build_datapackage.py b/digipipe/esys/scripts/build_datapackage.py
new file mode 100644
index 00000000..16085c09
--- /dev/null
+++ b/digipipe/esys/scripts/build_datapackage.py
@@ -0,0 +1,346 @@
+# coding: utf-8
+r"""
+Inputs
+-------
+scenario_specs : str
+ ``scenarios/{scenario}.yml``: path of input file (.yml) containing scenario
+ specifications
+destination : str
+ ``results/{scenario}/preprocessed``: path of output directory
+logfile : str
+ ``results/{scenario}/{scenario}.log``: path to logfile
+
+Outputs
+---------
+oemoflex.EnergyDatapackage
+ EnergyDatapackage that can be read by oemof.tabular, with data (scalars and
+ timeseries) as csv and metadata (describing resources and foreign key
+ relations) as json.
+
+Description
+-------------
+The script creates an empty EnergyDatapackage from the specifications given in
+the scenario_specs, fills it with scalar and timeseries data, infers the
+metadata and saves it to the given destination. Further, additional parameters
+like emission limit are saved in a separate file.
+
+Explanations about the structure of the preprocessed datapackage can be found
+in section :ref:`Build datapackages` of the
+`docu `_.
+
+"""
+import os
+import sys
+from collections import OrderedDict
+
+import pandas as pd
+from oemoflex import config as oemoflex_config
+from oemoflex.model.datapackage import EnergyDataPackage
+
+from digipipe.esys.esys.config import esys_conf
+from digipipe.esys.esys.config.esys_conf import load_yaml
+from digipipe.esys.esys.model import (
+ bus_attrs_update,
+ component_attrs_update,
+ foreign_keys_update,
+ model_structures,
+)
+from digipipe.esys.esys.tools.data_processing import (
+ expand_regions,
+ filter_df,
+ multi_load_b3_scalars,
+ multi_load_b3_timeseries,
+ prepare_attr_name,
+ save_df,
+ unstack_timeseries,
+ update_filtered_df,
+)
+
+
+def update_with_checks(old, new):
+ r"""
+ Updates a Series or DataFrame with new data. Raises a warning if there is
+ new data that is not in the index of the old data.
+
+ Parameters
+ ----------
+ old : pd.Series or pd.DataFrame
+ Old Series or DataFrame to update
+ new : pd.Series or pd.DataFrame
+ New Series or DataFrame
+
+ Returns
+ -------
+ None
+ """
+ # Check if some data would get lost
+ if not new.index.isin(old.index).all():
+ logger.warning("Index of new data is not in the index of old data.")
+
+ try:
+ # Check if it overwrites by setting errors = 'raise'
+ old.update(new, errors="raise")
+ except ValueError:
+ old.update(new, errors="ignore")
+ logger.warning("Update overwrites existing data.")
+
+
+def parametrize_scalars(edp, scalars, filters):
+ r"""
+ Parametrizes an oemoflex.EnergyDataPackage with scalars. Accepts an
+ OrderedDict of filters that is used to filter the scalars and
+ subsequently update the EnergyDatapackage.
+
+ Parameters
+ ----------
+ edp : oemoflex.EnergyDatapackage
+ EnergyDatapackage to parametrize
+ scalars : pd.DataFrame in oemof_B3-Resources format.
+ Scalar data
+ filters : OrderedDict
+ Filters for the scalar data
+
+ Returns
+ -------
+ edp : oemoflex.EnergyDatapackage
+ Parametrized EnergyDatapackage
+ """
+ edp.stack_components()
+
+ # apply filters subsequently
+ filtered = update_filtered_df(scalars, filters)
+
+ # set index to component name and var_name
+ filtered = filtered.set_index(["name", "var_name"]).loc[:, "var_value"]
+
+ # check if there are duplicates after setting index
+ duplicated = filtered.loc[filtered.index.duplicated()]
+
+ if duplicated.any():
+ raise ValueError(
+ f"There are duplicates in the scalar data: {duplicated}"
+ )
+
+ update_with_checks(edp.data["component"], filtered)
+
+ edp.unstack_components()
+
+ logger.info(f"Updated DataPackage with timeseries from '{paths_scalars}'.")
+
+ return edp
+
+
+def parametrize_sequences(edp, ts, filters):
+ r"""
+ Parametrizes an oemoflex.EnergyDataPackage with timeseries.
+
+ Parameters
+ ----------
+ edp : oemoflex.EnergyDatapackage
+ EnergyDatapackage to parametrize
+ ts : pd.DataFrame in oemof_B3-Resources format.
+ Timeseries data
+ filters : dict
+ Filters for timeseries data
+
+ Returns
+ -------
+ edp : oemoflex.EnergyDatapackage
+ Parametrized EnergyDatapackage
+ """
+ # Filter timeseries
+ _ts = ts.copy()
+
+ for key, value in filters.items():
+ _ts = filter_df(_ts, key, value)
+
+ # Group timeseries and parametrize EnergyDatapackage
+ ts_groups = _ts.groupby("var_name")
+
+ for name, group in ts_groups:
+
+ data = group.copy() # avoid pandas SettingWithCopyWarning
+
+ data.loc[:, "var_name"] = (
+ data.loc[:, "region"] + "-" + data.loc[:, "var_name"]
+ )
+
+ data_unstacked = unstack_timeseries(data)
+
+ edp.data[name] = data_unstacked
+
+ edp.data[name].index.name = "timeindex"
+
+ logger.info(
+ f"Updated DataPackage with timeseries from '{paths_timeseries}'."
+ )
+
+ return edp
+
+
+def load_additional_scalars(scalars, filters):
+ """Loads additional scalars like the emission limit and filters by
+ 'scenario_key'"""
+ # get electricity/gas relations and parameters for the calculation of
+ # emission_limit
+ el_gas_rel = scalars.loc[
+ scalars.var_name == esys_conf.settings.build_datapackage.el_gas_relation
+ ]
+ emissions = scalars.loc[
+ scalars.carrier == esys_conf.settings.build_datapackage.emission
+ ]
+
+ # get `output_parameters` of backpressure components as they are not taken
+ # into consideration in oemof.tabular so far. They are added to the
+ # components' output flow towards the heat bus in script `optimize.py`.
+ bpchp_out = scalars.loc[
+ (scalars.tech == "bpchp") & (scalars.var_name == "output_parameters")
+ ]
+
+ # concatenate data for filtering
+ df = pd.concat([el_gas_rel, emissions, bpchp_out])
+
+ # subsequently apply filters
+ filtered_df = update_filtered_df(df, filters)
+
+ # calculate emission limit and prepare data frame in case all necessary data
+ # is available
+ _filtered_df = filtered_df.copy().set_index("var_name")
+ try:
+ emission_limit = calculate_emission_limit(
+ _filtered_df.at["emissions_1990", "var_value"],
+ _filtered_df.at["emissions_not_modeled", "var_value"],
+ _filtered_df.at["emission_reduction_factor", "var_value"],
+ )
+ except KeyError:
+ emission_limit = None
+
+ emission_limit_df = pd.DataFrame(
+ {
+ "var_name": "emission_limit",
+ "var_value": emission_limit,
+ "carrier": "emission",
+ "var_unit": "kg_CO2_eq",
+ "scenario_key": "ALL",
+ },
+ index=[0],
+ )
+
+ # add emission limit to filtered additional scalars and adapt format of
+ # data frame
+ add_scalars = pd.concat([filtered_df, emission_limit_df], sort=False)
+ add_scalars.reset_index(inplace=True, drop=True)
+ add_scalars.index.name = "id_scal"
+
+ return add_scalars
+
+
+def save_additional_scalars(additional_scalars, destination):
+ """Saves `additional_scalars` to additional_scalar_file in `destination`"""
+ filename = os.path.join(
+ destination,
+ esys_conf.settings.build_datapackage.additional_scalars_file,
+ )
+ save_df(additional_scalars, filename)
+
+ logger.info(f"Saved additional scalars to '{filename}'.")
+
+
+def calculate_emission_limit(
+ emissions_1990, emissions_not_modeled, emission_reduction_factor
+):
+ """Calculates the emission limit.
+ Emission limit is calculated by
+ emissions_1990 * (1 - emission_reduction_factor) - emissions_not_modeled"""
+
+ return (
+ emissions_1990 * (1 - emission_reduction_factor) - emissions_not_modeled
+ )
+
+
+if __name__ == "__main__":
+ scenario_specs = sys.argv[1]
+
+ destination = sys.argv[2]
+
+ logger = esys_conf.add_snake_logger("build_datapackage")
+
+ scenario_specs = load_yaml(scenario_specs)
+
+ model_structure = model_structures[scenario_specs["model_structure"]]
+
+ oemoflex_config.config.settings.SEPARATOR = (
+ esys_conf.settings.general.separator
+ )
+
+ # setup empty EnergyDataPackage
+ datetimeindex = pd.date_range(
+ start=scenario_specs["datetimeindex"]["start"],
+ freq=scenario_specs["datetimeindex"]["freq"],
+ periods=scenario_specs["datetimeindex"]["periods"],
+ )
+
+ # setup default structure
+ edp = EnergyDataPackage.setup_default(
+ basepath=destination,
+ datetimeindex=datetimeindex,
+ bus_attrs_update=bus_attrs_update,
+ component_attrs_update=component_attrs_update,
+ name=scenario_specs["name"],
+ regions=model_structure["regions"],
+ links=model_structure["links"],
+ busses=model_structure["busses"],
+ components=model_structure["components"],
+ )
+
+ # parametrize scalars
+ paths_scalars = scenario_specs["paths_scalars"]
+
+ scalars = multi_load_b3_scalars(paths_scalars)
+
+ # Replace 'ALL' in the column regions by the actual regions
+ scalars = expand_regions(scalars, model_structure["regions"])
+
+ # Check and set attribute 'name'
+ scalars = prepare_attr_name(
+ scalars,
+ esys_conf.settings.build_datapackage.overwrite_name,
+ )
+
+ # get filters for scalars
+ filters = OrderedDict(sorted(scenario_specs["filter_scalars"].items()))
+
+ # load additional scalars like "emission_limit" and filter by `filters` in
+ # 'scenario_key'
+ additional_scalars = load_additional_scalars(
+ scalars=scalars, filters=filters
+ )
+
+ # Drop those scalars that do not belong to a specific component
+ scalars = scalars.loc[~scalars["name"].isna()]
+
+ # filter and parametrize scalars
+ edp = parametrize_scalars(edp, scalars, filters)
+
+ # parametrize timeseries
+ paths_timeseries = scenario_specs["paths_timeseries"]
+
+ ts = multi_load_b3_timeseries(paths_timeseries)
+
+ filters = scenario_specs["filter_timeseries"]
+
+ edp = parametrize_sequences(edp, ts, filters)
+
+ # save to csv
+ edp.to_csv_dir(destination)
+
+ logger.info(f"Saved datapackage to '{destination}'.")
+
+ save_additional_scalars(
+ additional_scalars=additional_scalars, destination=destination
+ )
+
+ logger.info(f"Saved additional_scalars to '{destination}'.")
+
+ # add metadata
+ edp.infer_metadata(foreign_keys_update=foreign_keys_update)
diff --git a/digipipe/esys/scripts/create_empty_scalars.py b/digipipe/esys/scripts/create_empty_scalars.py
new file mode 100644
index 00000000..acabbcbd
--- /dev/null
+++ b/digipipe/esys/scripts/create_empty_scalars.py
@@ -0,0 +1,209 @@
+# coding: utf-8
+r"""
+Inputs
+-------
+scenarios_dir : str
+ ``scenarios``: path to scenarios directory
+destination : str
+ ``raw/scalars/empty_scalars.csv``: path of output directory for empty
+ scalars of all scenarios
+
+Outputs
+---------
+pandas.DataFrame
+ Empty scalar in oemof-B3 resource format.
+
+Description
+-------------
+The script creates an empty DataFrame for scalar data that serve as a template
+for input data.
+"""
+import os
+import sys
+
+import pandas as pd
+from oemoflex.model.datapackage import EnergyDataPackage
+
+from digipipe.esys.esys import model
+from digipipe.esys.esys.config.esys_conf import load_yaml, settings
+from digipipe.esys.esys.model import bus_attrs_update, model_structures
+from digipipe.esys.esys.tools.data_processing import (
+ HEADER_B3_SCAL,
+ format_header,
+ load_b3_scalars,
+ save_df,
+ sort_values,
+)
+
+NON_REGIONAL = settings.create_empty_scalars.non_regional
+
+
+def get_edp_from_scenario(scenario_specs):
+ model_structure = model_structures[scenario_specs["model_structure"]]
+
+ # setup empty EnergyDataPackage
+ datetimeindex = pd.date_range(
+ start=scenario_specs["datetimeindex"]["start"],
+ freq=scenario_specs["datetimeindex"]["freq"],
+ periods=scenario_specs["datetimeindex"]["periods"],
+ )
+
+ # setup default structure
+ edp = EnergyDataPackage.setup_default(
+ basepath=destination,
+ datetimeindex=datetimeindex,
+ bus_attrs_update=bus_attrs_update,
+ component_attrs_update=component_attrs_update,
+ name=scenario_specs["name"],
+ regions=model_structure["regions"],
+ links=model_structure["links"],
+ busses=model_structure["busses"],
+ components=model_structure["components"],
+ )
+
+ edp.stack_components()
+
+ return edp
+
+
+def format_input_scalars(df):
+ _df = df.copy()
+
+ _df = format_header(_df, HEADER_B3_SCAL, "id_scal")
+
+ # Keep only those rows whose values are not set
+ if settings.create_empty_scalars.drop_default_scalars:
+ _df = _df.loc[_df.loc[:, "var_value"].isna()]
+
+ # Combine those parameters that are valid for all regions
+ _df.loc[_df["var_name"].isin(NON_REGIONAL), ["name", "region"]] = [
+ None,
+ "ALL",
+ ]
+
+ _df.drop_duplicates(inplace=True)
+
+ _df = sort_values(_df)
+
+ return _df
+
+
+def expand_scalars(df, column, where, expand):
+ _df = df.copy()
+
+ _df_cc = _df.loc[df[column] == where].copy()
+
+ _df_wo_cc = _df.loc[df[column] != where].copy()
+
+ for var in expand:
+
+ d = _df_cc.copy()
+
+ d[column] = var
+
+ _df_wo_cc = pd.concat([_df_wo_cc, d])
+
+ _df_wo_cc = sort_values(_df_wo_cc)
+
+ return _df_wo_cc
+
+
+def add_new_entry_to_scalars(sc, new_entry_dict):
+ sc = sc.append(new_entry_dict, ignore_index=True)
+
+ return sc
+
+
+def save_empty_scalars(sc, path):
+
+ if os.path.exists(path):
+ all_sc = load_b3_scalars(path)
+
+ if all_sc.empty:
+ all_sc = sc
+ else:
+ all_sc = all_sc.append(sc, ignore_index=True)
+ all_sc.index.name = sc.index.name
+
+ else:
+ all_sc = sc
+
+ all_sc.drop_duplicates(inplace=True)
+
+ save_df(all_sc, path)
+
+
+if __name__ == "__main__":
+ scenarios_dir = sys.argv[1]
+
+ destination = sys.argv[2]
+
+ scenarios = os.listdir(scenarios_dir)
+
+ for scenario_specs in scenarios:
+ component_attrs_update = load_yaml(
+ os.path.join(model.here, "component_attrs_update.yml")
+ )
+
+ scenario_specs = load_yaml(os.path.join(scenarios_dir, scenario_specs))
+
+ edp = get_edp_from_scenario(scenario_specs)
+
+ components = edp.data["component"].reset_index()
+
+ empty_scalars = format_input_scalars(components)
+
+ # set scenario name
+ empty_scalars.loc[:, "scenario_key"] = scenario_specs["name"]
+
+ # if empty raw scalars should be created, reverse the annuisation as
+ # well.
+ # if empty resources scalars are needed, set this to False.
+ raw_scalars = True
+ if raw_scalars:
+ empty_scalars = expand_scalars(
+ empty_scalars,
+ column="var_name",
+ where="capacity_cost",
+ expand=["capacity_cost_overnight", "lifetime", "fixom_cost"],
+ )
+
+ empty_scalars = expand_scalars(
+ empty_scalars,
+ column="var_name",
+ where="storage_capacity_cost",
+ expand=[
+ "storage_capacity_cost_overnight",
+ "storage_fixom_cost",
+ ],
+ )
+
+ # Add wacc
+ wacc_dict = {"scenario_key": scenario_specs["name"]}
+ wacc_dict.update(settings.create_empty_scalars.wacc)
+ empty_scalars = add_new_entry_to_scalars(empty_scalars, wacc_dict)
+
+ # Add emission limits
+ emissions_not_modeled = {"scenario_key": scenario_specs["name"]}
+ emissions_not_modeled.update(
+ settings.create_empty_scalars.emissions_not_modeled
+ )
+ empty_scalars = add_new_entry_to_scalars(
+ empty_scalars, emissions_not_modeled
+ )
+
+ emissions_1990 = {"scenario_key": scenario_specs["name"]}
+ emissions_1990.update(settings.create_empty_scalars.emissions_1990)
+ empty_scalars = add_new_entry_to_scalars(empty_scalars, emissions_1990)
+
+ emission_reduction_factor = {"scenario_key": scenario_specs["name"]}
+ emission_reduction_factor.update(
+ settings.create_empty_scalars.emission_reduction_factor
+ )
+ empty_scalars = add_new_entry_to_scalars(
+ empty_scalars, emission_reduction_factor
+ )
+
+ empty_scalars = sort_values(empty_scalars)
+
+ save_empty_scalars(empty_scalars, destination)
diff --git a/digipipe/esys/scripts/create_empty_ts.py b/digipipe/esys/scripts/create_empty_ts.py
new file mode 100644
index 00000000..88037b8a
--- /dev/null
+++ b/digipipe/esys/scripts/create_empty_ts.py
@@ -0,0 +1,360 @@
+# coding: utf-8
+r"""
+Inputs
+-------
+scenarios_dir : str
+ ``esys/scenarios``: path to scenarios directory
+destination : str
+ ``store/datasets/esys_raw/time_series/empty_ts_efficiencies.csv``: path of
+ output directory for empty ts with efficiencies of all scenarios
+ ``store/datasets/esys_raw/time_series/empty_ts_feedin.csv``: path of output
+ directory for empty ts with feedins of all scenarios
+ ``store/datasets/esys_raw/time_series/empty_ts_load.csv``: path of output
+ directory for empty ts with loads of all scenarios
+
+Outputs
+---------
+pandas.DataFrame
+ Empty ts in oemof-B3 resource format.
+
+Description
+-------------
+The script creates empty DataFrames for load, feed-in and efficiency time
+series data that serve as template for input data.
+"""
+import os
+import sys
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+from oemoflex.model.model_structure import module_path
+
+from digipipe.esys.esys import model
+from digipipe.esys.esys.config.esys_conf import load_yaml, settings
+from digipipe.esys.esys.model import model_structures
+from digipipe.esys.esys.tools.data_processing import (
+ HEADER_B3_TS,
+ stack_timeseries,
+)
+
+
+def get_sub_dict(subsub_key, _dict):
+ """
+ This function extracts a subsub-dictionary from a dictionary using a
+ subsub-key
+
+ Inputs
+ -------
+ subsub_key : str
+ Key of the subsub-dictionary
+
+ _dict : dict
+ Dictionary with two inner dictionaries
+
+ Outputs
+ -------
+ subsub_dict : dictionary
+ Subsub-dictionary
+
+ """
+
+ subsub_dict = [
+ subsubdict
+ for sub_dict in _dict.values()
+ for subsubdict in sub_dict.values()
+ if subsub_key in subsubdict
+ ]
+
+ return subsub_dict
+
+
+def create_empty_ts_with_zero_or_nan_values(periods, date_rng, name):
+ """
+ Returns a pandas DataFrame with time series values set to either zeros or
+ NaNs, based on settings.yaml
+
+ Parameters
+ ----------
+ periods : int
+ Number of periods in the time series
+ date_rng : pd.DatetimeIndex
+ Datetime index specifying start and end dates of the time series
+ name : str
+ Name of the time series column in the DataFrame
+
+ Returns
+ -------
+ df : pd.DataFrame
+ A pandas DataFrame containing the time series values, with 'name' as
+ the column name.
+
+ Raises
+ ------
+ KeyError
+ If settings.create_empty_ts.ts_values is not set to either 'zeros' or
+ 'empty'.
+ """
+ ts_values = settings.create_empty_ts.ts_values
+
+ if ts_values == "zeros":
+ df = pd.DataFrame(
+ np.zeros((periods, 1)), index=date_rng, columns=[name]
+ )
+ elif ts_values == "empty":
+ df = pd.DataFrame(
+ np.empty((periods, 1)) * np.nan, index=date_rng, columns=[name]
+ )
+ else:
+ raise KeyError(
+ f"{ts_values} is not a valid option. Valid options are: 'zeros' or"
+ f"'empty'. Please provide a valid value for ts_values in "
+ f"settings.yaml"
+ )
+
+ return df
+
+
+def create_empty_ts(name):
+ """
+ This function provides a Dataframe with a time series of zeros
+ according to the start, periods and freq of the scenario specifications
+
+ Parameters
+ ----------
+ name : str
+ Name of the ts
+
+ Returns
+ -------
+ df : Dataframe
+ Dataframe containing ts with zeros as values and name as column name
+
+ """
+ datetime_format = settings.create_empty_ts.datetime_format
+
+ # Get start date from scenario specifications
+ start = datetime.strptime(
+ scenario_specs["filter_timeseries"]["timeindex_start"], datetime_format
+ )
+
+ # Get periods and freq from scenario specifications
+ periods = scenario_specs["datetimeindex"]["periods"]
+ freq = scenario_specs["datetimeindex"]["freq"]
+
+ # Get date range from start periods and freq
+ date_rng = pd.date_range(start=start, periods=periods, freq=freq)
+
+ # Create DataFrame with ts of zeros from date range and name
+ df = create_empty_ts_with_zero_or_nan_values(periods, date_rng, name)
+
+ return df
+
+
+def get_df_of_all_empty_ts(profile_names, _region):
+ """
+ This function provides a Dataframe with all ts of a profile (load, feedin
+ or efficiency)
+
+ Inputs
+ -------
+ profile_names : list
+ List with names of profiles (loads, feedins or efficiencies)
+
+ _region : str
+ Region
+
+ Outputs
+ -------
+ ts_df : Dataframe
+ Dataframe with empty time series (consisting of zeros)
+
+ """
+ ts_df = pd.DataFrame(columns=HEADER_B3_TS)
+
+ for name in profile_names:
+ df = create_empty_ts(name)
+
+ # Stack Dataframe
+ stacked_df = stack_timeseries(df)
+
+ # Reindex according to time series in schema directory
+ stacked_df = stacked_df.reindex(columns=HEADER_B3_TS)
+
+ # Add region and scenario_specs to Dataframe
+ stacked_df["region"] = _region
+ stacked_df["scenario_key"] = settings.create_empty_ts.filter_ts
+
+ # Append Dataframe to ts_profile_df and add index name
+ ts_df = pd.concat([ts_df, stacked_df], ignore_index=True)
+ ts_df.index.name = "id_ts"
+
+ return ts_df
+
+
+def drop_duplicates(_df):
+ """
+ Remove duplicate rows from a pandas DataFrame based on specified columns.
+
+ Parameters
+ ----------
+ _df : pandas DataFrame
+ The DataFrame to remove duplicates from.
+
+ Returns
+ -------
+ _df : pandas DataFrame
+ The updated DataFrame with duplicate rows removed.
+
+ Notes
+ -----
+ Duplicate rows are determined based on the values in the specified columns.
+ By default, all columns except the "series" column are used to determine
+ duplicates. If there are multiple rows with the same values in the
+ specified columns, only the first occurrence is kept and subsequent
+ occurrences are dropped.
+ """
+ columns = [col for col in _df.columns if col != "series"]
+
+ _df = _df.drop_duplicates(
+ subset=columns,
+ ignore_index=True,
+ )
+
+ return _df
+
+
+def save_ts(_df, path):
+ """
+ This function saves time series that contain values of datetime format
+ Datetime format used: "%Y-%m-%d %H:%M:%S"
+
+ Parameters
+ ----------
+ _df : Dataframe
+ Dataframe with value(s) of datetime format
+
+ path : str
+ Path where df is saved to
+
+
+ """
+ if not os.path.exists(path) or settings.create_empty_ts.overwrite:
+
+ _df.index.name = "id_ts"
+
+ _df.to_csv(
+ path,
+ index=True,
+ date_format="%Y-%m-%d %H:%M:%S",
+ sep=settings.general.separator,
+ )
+
+
+if __name__ == "__main__":
+ scenarios_dir = sys.argv[1]
+
+ path_empty_load_ts = sys.argv[2]
+ path_empty_ts_feedin = sys.argv[3]
+ path_empty_ts_efficiencies = sys.argv[4]
+
+ scenarios = os.listdir(scenarios_dir)
+
+ all_load_ts = pd.DataFrame(columns=HEADER_B3_TS)
+ all_feedin_ts = pd.DataFrame(columns=HEADER_B3_TS)
+ all_efficiencies_ts = pd.DataFrame(columns=HEADER_B3_TS)
+
+ component_attrs_file = (os.path.join(module_path, "component_attrs.yml"),)
+ component_attrs = load_yaml(component_attrs_file[0])
+
+ for scenario_specs in scenarios:
+ scenario_specs = load_yaml(os.path.join(scenarios_dir, scenario_specs))
+ model_structure = model_structures[scenario_specs["model_structure"]]
+
+ component_attrs_update = load_yaml(
+ os.path.join(model.here, "component_attrs_update.yml")
+ )
+
+ # Get all foreign_keys that contain "profile" from
+ # component_attrs
+ foreign_keys_profile = get_sub_dict("profile", component_attrs)
+
+ # Get all foreign_keys that contain "profile" from
+ # component_attrs_update
+ foreign_keys_profile.extend(
+ get_sub_dict("profile", component_attrs_update)
+ )
+
+ # Drop the ones that are not listed in model_structure
+ foreign_keys_profile = [
+ fk
+ for fk in foreign_keys_profile
+ if fk["profile"].split("-profile")[0]
+ in model_structure["components"]
+ ]
+
+ # Get all foreign_keys that contain "efficiency" from
+ # component_attrs_update
+ foreign_keys_efficiency = get_sub_dict(
+ "efficiency", component_attrs_update
+ )
+
+ # Save profile names depending on whether it is a load, a feed-in or an
+ # efficiency
+ load_names = [
+ attr_subdict["profile"]
+ for attr_subdict in foreign_keys_profile
+ if "demand" in attr_subdict["profile"] # noqa: E713
+ ]
+ feedin_names = [
+ attr_subdict["profile"]
+ for attr_subdict in foreign_keys_profile
+ if not "demand" in attr_subdict["profile"] # noqa: E713
+ ]
+ efficiency_names = [
+ attr_subdict["efficiency"]
+ for attr_subdict in foreign_keys_efficiency
+ ]
+
+ ts_dict = {
+ "load": load_names,
+ "feedin": feedin_names,
+ "efficiency": efficiency_names,
+ }
+
+ for region in model_structure["regions"]:
+
+ if load_names:
+ df_loads = get_df_of_all_empty_ts(load_names, region)
+ all_load_ts = pd.concat(
+ [all_load_ts, df_loads], ignore_index=True
+ )
+
+ if feedin_names:
+ df_feedin = get_df_of_all_empty_ts(feedin_names, region)
+ all_feedin_ts = pd.concat(
+ [all_feedin_ts, df_feedin], ignore_index=True
+ )
+
+ if efficiency_names:
+ df_efficiencies = get_df_of_all_empty_ts(
+ efficiency_names, region
+ )
+ all_efficiencies_ts = pd.concat(
+ [all_efficiencies_ts, df_efficiencies], ignore_index=True
+ )
+
+ ts_dict = {
+ "load": (load_names, all_load_ts, path_empty_load_ts),
+ "feedin": (feedin_names, all_feedin_ts, path_empty_ts_feedin),
+ "efficiency": (
+ efficiency_names,
+ all_efficiencies_ts,
+ path_empty_ts_efficiencies,
+ ),
+ }
+
+ for ts_type, (ts_name, ts_data, ts_path) in ts_dict.items():
+ if ts_name:
+ ts_data = drop_duplicates(ts_data)
+ save_ts(ts_data, ts_path)
diff --git a/digipipe/esys/scripts/optimize.py b/digipipe/esys/scripts/optimize.py
new file mode 100644
index 00000000..53ae65e5
--- /dev/null
+++ b/digipipe/esys/scripts/optimize.py
@@ -0,0 +1,329 @@
+# coding: utf-8
+r"""
+Inputs
+-------
+preprocessed : str
+ ``results/{scenario}/preprocessed``: Path to preprocessed EnergyDatapackage
+ containing elements, sequences and datapackage.json.
+optimized : str
+ ``results/{scenario}/optimized/`` Target path to store dump of
+ oemof.solph.Energysystem with optimization results and parameters.
+logfile : str
+ ``results/{scenario}/{scenario}.log``: path to logfile
+
+Outputs
+---------
+es.dump
+ oemof.solph.EnergySystem with results, meta-results and parameters
+
+Description
+-------------
+Given an EnergyDataPackage, this script creates an oemof.solph.EnergySystem and
+an oemof.solph.Model, which is optimized.
+
+The following constraints are added:
+ - `emission_limit`: maximum amount of emissions
+ - `equate_flows_by_keyword`: electricity-gas relation is set
+ (electricity/gas = factor).
+ This constraint is only added if 'electricity_gas_relation' is added to
+ the scalars.
+ To use this constraint you need to copy
+ [`equate_flows.py`](https://github.com/oemof/oemof-solph/blob/features/
+ equate-flows/src/oemof/solph/constraints/equate_variables.py)
+ of oemof.solph into `/tools` directory of `oemof-B3`.
+
+The EnergySystem with results, meta-results and parameters is saved.
+"""
+import logging
+import os
+import sys
+
+import numpy as np
+from oemof import solph
+from oemof.solph import EnergySystem, Model, constraints, processing
+
+# DONT REMOVE THIS LINE!
+# pylint: disable=unusedimport
+from oemof.tabular import datapackage # noqa
+from oemof.tabular.facades import TYPEMAP
+
+from digipipe.esys.esys.config import esys_conf
+from digipipe.esys.esys.tools import data_processing as dp
+from digipipe.esys.esys.tools.equate_flows import equate_flows_by_keyword
+from digipipe.esys.esys.tools.timing import Timer
+
+logger = logging.getLogger()
+
+
+def drop_values_by_keyword(df, keyword="None"):
+ """drops row if `var_value` is None"""
+ drop_indices = df.loc[df.var_value == keyword].index
+ df = df.drop(drop_indices)
+ return df
+
+
+def get_emission_limit(scalars):
+ """
+ Gets emission limit from scalars and returns None if it is missing or
+ None.
+ """
+ emission_df_raw = scalars.loc[scalars["carrier"] == "emission"].set_index(
+ "var_name"
+ )
+ emission_df = drop_values_by_keyword(emission_df_raw)
+
+ # return None if no emission limit is given ('None' or entry missing)
+ if (
+ emission_df.empty
+ or emission_df.at["emission_limit", "var_value"] is np.nan
+ ):
+ logger.info("No emission limit will be set.")
+ return None
+ else:
+ limit = emission_df.at["emission_limit", "var_value"]
+ logger.info(f"Emission limit will be set to {limit}.")
+ return limit
+
+
+def get_electricity_gas_relations(scalars):
+ r"""
+ Gets electricity/gas relations from scalars. Returns None if no relations
+ are given.
+
+ Returns
+ -------
+ pd.DataFrame
+ Contains rows of scalars with 'var_name' `EL_GAS_RELATION`.
+ If no relation is given returns None.
+ """
+ relations_raw = scalars.loc[
+ scalars.var_name == esys_conf.settings.optimize.el_gas_relation
+ ]
+ # drop relations that are None
+ relations = drop_values_by_keyword(relations_raw)
+ if relations.empty:
+ logger.info("No gas electricity relation will be set.")
+ return None
+ else:
+ busses = relations.carrier.drop_duplicates().values
+ logger.info(
+ f"Gas electricity relations will be set for busses: {busses}"
+ )
+ return relations
+
+
+def get_bpchp_output_parameters(scalars):
+ r"""Gets 'output_parameters' of backpressure CHPs from scalars and
+ returns None if it is missing or None."""
+ bpchp_outs_raw = scalars.loc[
+ (scalars.tech == "bpchp") & (scalars.var_name == "output_parameters")
+ ]
+
+ # drop rows that have empty dict as var_value
+ bpchp_outs = drop_values_by_keyword(bpchp_outs_raw, "{}")
+ if bpchp_outs.empty:
+ return None
+ else:
+ return bpchp_outs
+
+
+def add_output_parameters_to_bpchp(parameters, energysystem):
+ r"""
+ Adds keywords for electricity-gas relation constraint to backpressure CHPs.
+
+ This is necessary as oemof.tabular does not support `output_parameters` of
+ these components, yet. The keywords are set as attributes of the output
+ flow towards `heat_bus`.
+
+ Parameters
+ ----------
+ parameters : pd.DataFrame
+ Contains output_parameters of backpressure CHP scalars.
+ energysystem : oemof.solph.network.EnergySystem
+ The energy system
+ """
+ # rename column 'name' as is collides with iterrows()
+ parameters.rename(columns={"name": "name_"}, inplace=True)
+ for i, element in parameters.iterrows():
+ if element.name_ in energysystem.groups:
+ # get heat bus the component is connected to
+ bus = energysystem.groups[element.name_].heat_bus
+
+ # get keyword and boolean value
+ split_str = element.var_value.split('"')
+ keyword = split_str[1]
+ value = bool(split_str[2].split("}")[0].split()[1])
+
+ # set keyword as attribute with value
+ setattr(
+ energysystem.groups[element.name_].outputs.data[bus],
+ keyword,
+ value,
+ )
+ else:
+ logging.warning(
+ f"No element '{element.name_}' in EnergySystem. Cannot add "
+ f"output_parameters."
+ )
+
+ return energysystem
+
+
+def add_electricity_gas_relation_constraints(model, relations):
+ r"""
+ Adds constraint `equate_flows_by_keyword` to `model`.
+
+ The components belonging to 'electricity' or 'gas' are selected by
+ keywords. The keywords of components powered by gas start with
+ `esys_conf.settings.optimize.gas_key` and such powered by electricity with
+ `esys_conf.settings.optimize.el_key`, followed by `carrier` and `region`
+ e.g. <`GAS_KEY`>--.
+
+ Parameters
+ ----------
+ model : oemof.solph.Model
+ optmization model
+ relations : pd.DataFrame
+ Contains electricity/gas relations in column 'var_value'. Further
+ contains at least columns 'carrier' and 'region'.
+ """
+ for index, row in relations.iterrows():
+ # Formulate suffix for keywords -
+ suffix = f"{row.carrier}-{row.region}"
+ equate_flows_by_keyword(
+ model=model,
+ keyword1=f"{esys_conf.settings.optimize.gas_key}-{suffix}",
+ keyword2=f"{esys_conf.settings.optimize.el_key}-{suffix}",
+ factor1=row.var_value,
+ name=f"equate_flows_{suffix}",
+ )
+
+
+def get_additional_scalars():
+ """
+ Returns additional scalars as pd.DataFrame or None if file does not exist
+ """
+ filename_add_scalars = os.path.join(preprocessed, "additional_scalars.csv")
+ if os.path.exists(filename_add_scalars):
+ scalars = dp.load_b3_scalars(filename_add_scalars)
+ return scalars
+ else:
+ return None
+
+
+if __name__ == "__main__":
+ preprocessed = sys.argv[1]
+
+ optimized = sys.argv[2]
+
+ logfile = sys.argv[3]
+ logger = esys_conf.add_snake_logger("optimize")
+
+ # get additional scalars, set to None at first
+ emission_limit = None
+ el_gas_relations = None
+ bpchp_out = None
+ additional_scalars = get_additional_scalars()
+ if additional_scalars is not None:
+ emission_limit = get_emission_limit(additional_scalars)
+ el_gas_relations = get_electricity_gas_relations(additional_scalars)
+ bpchp_out = get_bpchp_output_parameters(additional_scalars)
+
+ if not os.path.exists(optimized):
+ os.mkdir(optimized)
+
+ try:
+
+ logger.info(
+ f"Created solph.EnergSystem using oemof.solph version "
+ f"'{solph.__version__}'."
+ )
+
+ with Timer(text="Created solph.Energystem.", logger=logger.info):
+ es = EnergySystem.from_datapackage(
+ os.path.join(
+ preprocessed, esys_conf.settings.optimize.filename_metadata
+ ),
+ attributemap={},
+ typemap=TYPEMAP,
+ )
+
+ # Reduce number of timestep for debugging
+ if esys_conf.settings.optimize.debug:
+ es.timeindex = es.timeindex[:3]
+
+ logger.info(
+ "Using DEBUG mode: Running model with first 3 timesteps only."
+ )
+
+ # add output_parameters of bpchp
+ if bpchp_out is not None:
+ es = add_output_parameters_to_bpchp(
+ parameters=bpchp_out, energysystem=es
+ )
+
+ # create model from energy system (this is just oemof.solph)
+ logger.info("Creating solph.Model.")
+
+ with Timer(text="Created solph.Model.", logger=logger.info):
+ m = Model(es)
+
+ # add constraints
+ logger.info("Setting constraints.")
+
+ if emission_limit is not None:
+ constraints.emission_limit(m, limit=emission_limit)
+ if el_gas_relations is not None:
+ add_electricity_gas_relation_constraints(
+ model=m, relations=el_gas_relations
+ )
+
+ # tell the model to get the dual variables when solving
+ if esys_conf.settings.optimize.receive_duals:
+ m.receive_duals()
+
+ # save solver log to scenario specific location
+ solve_kwargs = esys_conf.settings.optimize.solve_kwargs
+ solve_kwargs["logfile"] = (
+ logfile.split("." + logfile.split(".")[-1])[0] + "_solver_log.log"
+ )
+
+ logger.info(
+ f"Solving with solver '{esys_conf.settings.optimize.solver}' "
+ f"using solve_kwargs '{esys_conf.settings.optimize.solve_kwargs}' "
+ f"and cmdline_options "
+ f"'{esys_conf.settings.optimize.cmdline_options}'."
+ )
+
+ with Timer(text="Solved the model.", logger=logger.info):
+ if esys_conf.settings.optimize.write_lp_file:
+ m.write(
+ os.path.join(optimized, "optimized.lp"),
+ io_options={"symbolic_solver_labels": True},
+ )
+ m.solve(
+ solver=esys_conf.settings.optimize.solver,
+ solve_kwargs=esys_conf.settings.optimize.solve_kwargs,
+ cmdline_options=esys_conf.settings.optimize.cmdline_options,
+ )
+
+ except: # noqa: E722
+ logger.exception(
+ f"Could not optimize energysystem for datapackage from "
+ f"'{preprocessed}'."
+ )
+ raise
+
+ else:
+
+ logger.info("Model solved. Collecting results.")
+
+ # get results from the solved model(still oemof.solph)
+ es.meta_results = processing.meta_results(m)
+ es.results = processing.results(m)
+ es.params = processing.parameter_as_dict(es)
+
+ # dump the EnergySystem
+ es.dump(optimized)
+
+ logger.info(f"Results saved to {optimized}.")
diff --git a/digipipe/esys/scripts/postprocess.py b/digipipe/esys/scripts/postprocess.py
new file mode 100644
index 00000000..157d3aba
--- /dev/null
+++ b/digipipe/esys/scripts/postprocess.py
@@ -0,0 +1,73 @@
+# coding: utf-8
+r"""
+Inputs
+-------
+optimized : str
+ ``results/{scenario}/optimized``: Directory containing dump of
+ oemof.solph.Energysystem with optimization results and parameters.
+scenario_name : str
+ ``{scenario}``: Name of the scenario.
+destination : str
+ ``results/{scenario}/postprocessed``: Target path for postprocessed
+ results.
+logfile : str
+ ``results/{scenario}/{scenario}.log``: path to logfile
+
+Outputs
+---------
+oemoflex.ResultsDatapackage
+ ResultsDatapackage
+
+Description
+-------------
+The script performs the postprocessing of optimization results.
+
+Explanations about the structure of the postprocessed data can be found in
+section :ref:`Postprocessing` of the
+`docu `_.
+"""
+import os
+import sys
+
+import pandas as pd
+from oemof.solph import EnergySystem
+from oemoflex import config as oemoflex_config
+from oemoflex.model.datapackage import ResultsDataPackage
+
+from digipipe.esys.esys.config import esys_conf
+
+if __name__ == "__main__":
+
+ optimized = sys.argv[1]
+
+ scenario_name = sys.argv[2]
+
+ destination = sys.argv[3]
+
+ logger = esys_conf.add_snake_logger("postprocess")
+
+ oemoflex_config.config.settings.SEPARATOR = (
+ esys_conf.settings.general.separator
+ )
+
+ try:
+ es = EnergySystem()
+
+ es.restore(optimized)
+
+ rdp = ResultsDataPackage.from_energysytem(es)
+
+ rdp.set_scenario_name(scenario_name)
+
+ rdp.to_csv_dir(destination)
+
+ pd.Series({"objective": es.meta_results["objective"]}).to_csv(
+ os.path.join(destination, "objective.csv"),
+ sep=esys_conf.settings.general.separator,
+ )
+
+ except: # noqa: E722
+ logger.exception(
+ f"Could not postprocess data from energysystem in '{optimized}'."
+ )
+ raise
diff --git a/digipipe/esys/scripts/prepare_scalars.py b/digipipe/esys/scripts/prepare_scalars.py
new file mode 100644
index 00000000..eb98f2cf
--- /dev/null
+++ b/digipipe/esys/scripts/prepare_scalars.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+r"""
+Inputs
+-------
+in_path1 : str
+ ``raw/scalars/costs_efficiencies.csv``: path incl. file name of input file
+ with raw scalar data
+out_path : str
+ ``results/_resources/scal_costs_efficiencies.csv``: path incl. file name of
+ output file with prepared scalar data
+
+Outputs
+---------
+pandas.DataFrame
+ with scalar data prepared for parametrization
+
+Description
+-------------
+The script performs the following steps to prepare scalar data for
+parametrization:
+
+* Calculate annualized investment cost from overnight cost, lifetime and wacc.
+"""
+
+import sys
+
+from oemof.tools.economics import annuity
+
+from digipipe.esys.esys.config import esys_conf
+from digipipe.esys.esys.tools.data_processing import (
+ ScalarProcessor,
+ load_b3_scalars,
+ save_df,
+)
+
+
+def annuise_investment_cost(sc):
+
+ for var_name_cost, var_name_fixom_cost in [
+ ("capacity_cost_overnight", "fixom_cost"),
+ ("storage_capacity_cost_overnight", "storage_fixom_cost"),
+ ]:
+
+ invest_data = sc.get_unstacked_var(
+ [var_name_cost, "lifetime", var_name_fixom_cost]
+ )
+
+ # TODO: Currently, (storage)_capacity_overnight_cost,
+ # (storage)_fixom_cost and lifetime have to be given for each tech and
+ # each scenario, but wacc may change per scenario, but is defined for
+ # all techs uniformly. Could offer a more general and flexible solution.
+
+ # wacc is defined per scenario, ignore other index levels
+ wacc = sc.get_unstacked_var("wacc")
+ wacc.index = wacc.index.get_level_values("scenario_key")
+
+ # set wacc per scenario_key
+ scenario_keys = invest_data.index.get_level_values("scenario_key")
+ invest_data["wacc"] = wacc.loc[scenario_keys].values
+
+ annuised_investment_cost = invest_data.apply(
+ lambda x: annuity(x[var_name_cost], x["lifetime"], x["wacc"])
+ + x[var_name_fixom_cost],
+ 1,
+ )
+
+ sc.append(
+ var_name_cost.replace("_overnight", ""), annuised_investment_cost
+ )
+
+ sc.drop(
+ [
+ "wacc",
+ "lifetime",
+ "capacity_cost_overnight",
+ "storage_capacity_cost_overnight",
+ "fixom_cost",
+ "storage_fixom_cost",
+ ]
+ )
+
+
+if __name__ == "__main__":
+ in_path = sys.argv[1] # path to raw scalar data
+ out_path = sys.argv[2] # path to destination
+
+ df = load_b3_scalars(in_path)
+
+ sc = ScalarProcessor(df)
+
+ annuise_investment_cost(sc)
+
+ sc.scalars = sc.scalars.sort_values(
+ by=["carrier", "tech", "var_name", "scenario_key"]
+ )
+
+ sc.scalars.reset_index(inplace=True, drop=True)
+
+ sc.scalars.index.name = esys_conf.settings.general.scal_index_name
+
+ save_df(sc.scalars, out_path)
diff --git a/digipipe/esys/scripts/write_costs_efficiencies.py b/digipipe/esys/scripts/write_costs_efficiencies.py
new file mode 100644
index 00000000..053f3ba9
--- /dev/null
+++ b/digipipe/esys/scripts/write_costs_efficiencies.py
@@ -0,0 +1,128 @@
+# coding: utf-8
+r"""
+This module provides functionality to map raw scalar data on costs and
+efficiencies from "store/raw/technology_data/data/raw_costs_efficiencies.csv" to
+the default costs and efficiencies in
+"store/datasets/esys_raw/data/scalars/default_costs_efficiencies.csv".
+The default file is initially created using the "write_default_scalars" rule.
+
+The mapped costs and efficiencies are then written to
+"store/datasets/esys_raw/data/scalars/costs_efficiencies.csv", and the original
+"default_costs_efficiencies.csv" file is deleted.
+"""
+import os
+import sys
+
+from digipipe.esys.esys.config.esys_conf import add_snake_logger, settings
+from digipipe.esys.esys.tools.data_processing import (
+ load_b3_scalars,
+ multi_filter_df,
+ save_df,
+)
+
+
+def check_var_value_empty(df, cols):
+ """
+ Checks if specified columns in a DataFrame have empty values.
+
+ Parameters
+ ----------
+ df : pandas.DataFrame
+ The DataFrame to check.
+ cols : list
+ List of column names to check.
+
+ Raises
+ ------
+ ValueError
+ If one or more columns contain non-empty values.
+
+ """
+ for col in cols:
+ if not (df[col].isna() | (df[col] == "None")).all():
+ raise ValueError(
+ "There are not empty values in one or more of the columns "
+ f"'{cols}' of {path_default_costs_eff}.\n"
+ f"Please make sure that the columns '{cols}' are empty."
+ )
+
+
+def map_var_value_costs_effs(df_1, df_2, cols):
+ """
+ Maps values from df_2 to df_1 based on matching variable names, carriers,
+ and technologies.
+
+ Parameters
+ ----------
+ df_1 : pandas.DataFrame
+ The source DataFrame where values will be mapped.
+ df_2 : pandas.DataFrame
+ The reference DataFrame containing the values to map.
+ cols : list
+ List of column names in df_1 to update with mapped values.
+
+ Returns
+ -------
+ pandas.DataFrame
+ The modified df_1 DataFrame with mapped values.
+
+ Raises
+ ------
+ ValueError
+ If a matching combination of var_name, carrier, and tech is not found
+ in df_2.
+
+ """
+
+ for index, row in df_1.iterrows():
+ var_name = row["var_name"]
+ carrier = row["carrier"]
+ tech = row["tech"]
+ df_3 = multi_filter_df(
+ df_2, var_name=var_name, carrier=carrier, tech=tech
+ )
+
+ if not df_3.empty:
+ for col in cols:
+ df_1.loc[index, col] = df_3[col].values[0]
+ else:
+ raise ValueError(
+ f"Value of var_name '{var_name}', carrier '{carrier}' and tech "
+ f"'{tech}' is missing in {path_raw_costs_eff}."
+ )
+
+ return df_1
+
+
+if __name__ == "__main__":
+ path_default_costs_eff = sys.argv[1]
+ path_raw_costs_eff = sys.argv[2]
+ path_costs_eff = sys.argv[3]
+
+ logger = add_snake_logger("data_processing")
+
+ default_costs_eff = load_b3_scalars(path_default_costs_eff)
+ raw_costs_eff = load_b3_scalars(path_raw_costs_eff)
+
+ # Name of Columns to be filled
+ target_cols = ["var_value", "var_unit", "source", "comment"]
+
+ # Check if no values written in target_cols of
+ # default_costs_efficiencies.csv
+ check_var_value_empty(default_costs_eff, target_cols)
+
+ # Map values from raw_costs_efficiencies.csv with
+ # default_costs_efficiencies.csv
+ costs_eff = map_var_value_costs_effs(
+ default_costs_eff, raw_costs_eff, target_cols
+ )
+
+ # Save costs_efficiencies.csv to store/datasets/esys_raw/data/scalars
+ save_df(costs_eff, path_costs_eff)
+
+ # Delete file default_costs_efficiencies.csv
+ if settings.write_costs_efficiencies.delete_default:
+ if os.path.exists(path_default_costs_eff):
+ os.remove(path_default_costs_eff)
+ # Print user info
+ logger.info(f"The file {path_default_costs_eff} has been deleted.")
diff --git a/digipipe/esys/scripts/write_default_scalars.py b/digipipe/esys/scripts/write_default_scalars.py
new file mode 100644
index 00000000..6caf852e
--- /dev/null
+++ b/digipipe/esys/scripts/write_default_scalars.py
@@ -0,0 +1,236 @@
+# coding: utf-8
+r"""
+This module contains functionality to update empty scalars (generated from
+scenarios with create_empty_scalars module) with default data and saves it to a
+new csv file.
+"""
+
+import sys
+
+import numpy as np
+import pandas as pd
+
+from digipipe.esys.esys.config.esys_conf import write_default_scalars
+from digipipe.esys.esys.tools.data_processing import (
+ filter_df,
+ load_b3_scalars,
+ save_df,
+)
+
+
+def clear_input_parameters_of_storages(_df):
+ """
+ Deletes empty dictionaries from "var_value" where "var_name" is
+ "input_parameters" and "type" is "storage". This allows default values to
+ be written to these fields.
+
+ Parameters
+ ----------
+ _df : pandas.DataFrame
+ DataFrame containing the data.
+
+ Returns
+ -------
+ None
+ This function modifies the "_df" DataFrame in-place.
+
+ """
+ empty_sc_df.loc[
+ (empty_sc_df["var_name"] == "input_parameters")
+ & (empty_sc_df["type"] == "storage"),
+ "var_value",
+ ] = None
+
+
+def get_var_value_and_comment(which):
+ """
+ Returns the variable value and a comment based on the input argument.
+
+ Parameters
+ ----------
+ which : str
+ string indicating what var_value and comment are chosen as default.
+ Valid options are: zeros, empty, high_costs, false and empty_dict.
+
+ Returns
+ -------
+ var_value : object
+ The updated variable value
+ comment : str
+ A comment for the var_value update
+ """
+ if which == "zeros":
+ var_value = 0
+ comment = "Zero"
+ elif which == "empty":
+ var_value = np.nan
+ comment = "Empty"
+ elif which == "high_costs":
+ var_value = 1000000000
+ comment = "High slack cost on shortage"
+ elif which == "false":
+ var_value = False
+ comment = "Empty"
+ elif which == "empty_dict":
+ var_value = "{}"
+ comment = "Empty"
+ elif which == "variable_costs":
+ var_value = '{"variable_costs": 1e-7}'
+ comment = "Own assumption to prevent hidden curtailment"
+ elif which == "emissions_not_modeled":
+ var_value = 0
+ comment = "No fossils in 2045 in the modelled sectors."
+ elif which == "emission_reduction_factor":
+ var_value = 1
+ comment = "100 % of GHG reduction in 2045"
+ else:
+ raise ValueError(
+ f"'{which}' is not a valid option. Please provide a valid options. "
+ f"Valid options are: zeros, empty, high_costs, false and "
+ f"empty_dict."
+ )
+
+ return var_value, comment
+
+
+def get_filter_condition(_df, _var_name, _type, _tech):
+ """
+ Returns a filter condition based on the input arguments.
+
+ Parameters
+ ----------
+ _df : pd.DataFrame
+ DataFrame containing the empty scalar data
+ _var_name : str
+ Name of the variable to be updated
+ _type : str
+ Type to be updated
+ _tech : str
+ Technology to be updated
+
+ Returns
+ -------
+ condition : pd.Series
+ A Boolean series used to filter the DataFrame to update the specified
+ variable
+ """
+ precondition = _df["var_value"].isna()
+ if (_type != "None") and (_tech != "None"):
+ condition = (
+ precondition
+ & (_df["var_name"] == _var_name)
+ & (_df["type"] == _type)
+ & (_df["tech"] == _tech)
+ )
+ elif (_type != "None") and (_tech == "None"):
+ condition = (
+ precondition
+ & (_df["var_name"] == _var_name)
+ & (_df["type"] == _type)
+ )
+ elif (_type == "None") and (_tech != "None"):
+ condition = (
+ precondition
+ & (_df["var_name"] == _var_name)
+ & (_df["tech"] == _tech)
+ )
+ else:
+ condition = precondition & (_df["var_name"] == _var_name)
+
+ return condition
+
+
+def update_df(_df, which, condition, unit):
+ """
+ Update the input DataFrame by setting values for specified columns based on
+ the provided condition.
+
+ Parameters
+ ----------
+ _df : pandas.DataFrame
+ The DataFrame to update.
+ which : str
+ Specifies the type of value to set for the "var_value" column.
+ Valid options are "zeros", "empty", "high_costs", "false", and
+ "empty_dict".
+ condition : pandas.Series of bools
+ A boolean mask indicating which rows of `_df` should be updated.
+ unit : str
+ The unit to write into the "var_unit" column.
+
+ Returns
+ -------
+ pandas.DataFrame
+ The updated DataFrame.
+ """
+ var_value, comment = get_var_value_and_comment(which)
+
+ _df.loc[condition, "var_value"] = var_value
+ _df.loc[condition, "comment"] = comment
+ _df.loc[condition, "var_unit"] = unit
+
+ return _df
+
+
+if __name__ == "__main__":
+ path_empty_sc = sys.argv[1]
+ path_default_sc = sys.argv[2]
+ path_default_costs_eff = sys.argv[3]
+
+ empty_sc_df = load_b3_scalars(path_empty_sc)
+
+ clear_input_parameters_of_storages(empty_sc_df)
+
+ write_empty_scalars_dict = write_default_scalars.write_default_scalars
+
+ for key, value in write_empty_scalars_dict.items():
+
+ condition = get_filter_condition(
+ empty_sc_df, value["var_name"], value["type"], value["tech"]
+ )
+ df_updated = update_df(
+ empty_sc_df, value["which"], condition, value["var_unit"]
+ )
+
+ # Get all unique values for var_name
+ var_names_all = list(df_updated["var_name"].unique())
+ # Get all values for var_name in context of costs and efficiencies
+ var_names_costs_efficiencies = list(
+ write_default_scalars.costs_efficiencies
+ )
+ # Get all remaining var_values needed for default_scalars.csv
+ var_names_scalars = [
+ var_name
+ for var_name in var_names_all
+ if var_name not in var_names_costs_efficiencies
+ ]
+
+ df_costs_efficiencies = filter_df(
+ df_updated, "var_name", var_names_costs_efficiencies
+ )
+
+ # Get all already by default set values of costs and efficiencies
+ df_default_costs_efficiencies = df_costs_efficiencies.dropna(
+ subset=["var_value"]
+ )
+
+ # Keep only non default values of costs and efficiencies
+ df_costs_efficiencies = df_costs_efficiencies[
+ df_costs_efficiencies["var_value"].isna()
+ ]
+
+ # Get remaining scalars
+ df_scalars = filter_df(df_updated, "var_name", var_names_scalars)
+
+ # Append all values set by default of costs and efficiencies to remaining
+ # scalars
+ df_scalars = pd.concat(
+ [df_scalars, df_default_costs_efficiencies], ignore_index=False
+ )
+
+ # Write all attributes attached to costs and efficiencies in separate
+ # default_cost_efficiencies.csv file
+ save_df(df_costs_efficiencies, path_default_costs_eff)
+
+ # Write all other scalars in default_scalars.csv
+ save_df(df_scalars, path_default_sc)
diff --git a/digipipe/esys/scripts/write_ts.py b/digipipe/esys/scripts/write_ts.py
new file mode 100644
index 00000000..94f6d52b
--- /dev/null
+++ b/digipipe/esys/scripts/write_ts.py
@@ -0,0 +1,229 @@
+# coding: utf-8
+r"""
+This module provides functionality to map time series from the subdirectories
+in "store/datasets/" to the empty time series in
+"store/datasets/esys_raw/data/time_series/empty_ts_efficiencies.csv",
+"store/datasets/esys_raw/data/time_series/empty_ts_feedin.csv" and
+"store/datasets/esys_raw/data/time_series/empty_ts_load.csv".
+The empty files are initially created using the "create_empty_ts" rule.
+
+The mapped time series are then written to
+"store/datasets/esys_raw/data/scalars/ts_efficiencies.csv",
+"store/datasets/esys_raw/data/scalars/ts_feedin.csv" and
+"store/datasets/esys_raw/data/scalars/ts_load.csv".
+
+The original files with the empty time series are deleted.
+"""
+
+import os
+import sys
+
+import pandas as pd
+
+from digipipe.esys.esys.config.esys_conf import add_snake_logger, map_ts
+from digipipe.esys.esys.tools.data_processing import (
+ HEADER_B3_TS,
+ load_b3_timeseries,
+ save_df,
+ stack_timeseries,
+ unstack_timeseries,
+)
+from digipipe.store.utils import get_abs_dataset_path
+
+
+def check_if_ts_name_in_maps_ts(ts_name, mapping_file_name):
+ """
+ Check if a time series name is present in the mapping of time series.
+
+ Parameters
+ ----------
+ ts_name : str
+ The name of the time series to check.
+ mapping_file_name : str
+ The name of the mapping file for reference in the error message.
+
+ Raises
+ ------
+ ValueError
+ If the time series name is missing in the mapping.
+
+ """
+ if (
+ ts_name not in map_ts.efficiencies
+ and ts_name not in map_ts.feedin
+ and ts_name not in map_ts.load
+ ):
+ raise ValueError(
+ f"Missing mapping of time series '{ts_name}'. "
+ f"Please provide a key and value for each time "
+ f"series with the file '{mapping_file_name}'."
+ )
+
+
+def get_datasets_data_name(key, which):
+ """
+ Retrieve the specific data value from the time series based on the provided
+ key and description.
+
+ Parameters
+ ----------
+ key : str
+ The name of the data to retrieve.
+ which : str
+ Specifies the kind of time series data.
+ Valid options are "efficiencies", "feedin", and "load".
+
+ Returns
+ -------
+ value : object
+ The data value corresponding to the given key and description.
+
+ Raises
+ ------
+ ValueError
+ If an invalid option is provided for the 'which' parameter.
+
+ """
+ if which == "efficiencies":
+ value = map_ts.efficiencies[key]
+ elif which == "feedin":
+ value = map_ts.feedin[key]
+ elif which == "load":
+ value = map_ts.load[key]
+ else:
+ raise ValueError(
+ "Please provide a valid description of the time series with "
+ f"'which'. '{which}' is not a valid option."
+ )
+
+ return value
+
+
+def check_file_exists(path, file_name, df, mapping_file_name):
+ """
+ Check if a file exists at the specified path and raise a FileNotFoundError
+ if not.
+
+ Parameters
+ ----------
+ path : str
+ The path to the file.
+ file_name : str
+ The name of the file (without the file extension).
+ df : pd.DataFrame
+ The DataFrame containing the variable name to be mapped.
+ mapping_file_name : str
+ The name of the mapping file for reference in the error message.
+
+ Raises
+ ------
+ FileNotFoundError
+ If the file does not exist at the specified path.
+
+ """
+ if not path:
+ raise FileNotFoundError(
+ f"The file '{file_name}.csv' could not be found. Please provide a "
+ f"valid file name with '{mapping_file_name}' to map "
+ f"'{df['var_name']}'."
+ )
+
+
+def write_ts_data(stacked_ts, data):
+ """
+ Write time series data to the specified stacked time series DataFrame.
+
+ Parameters
+ ----------
+ stacked_ts : pd.DataFrame
+ The stacked time series DataFrame.
+ data : pd.DataFrame
+ The data to write into the stacked time series.
+
+ Returns
+ -------
+ pd.DataFrame
+ The updated stacked time series DataFrame.
+
+ """
+ unstacked_data = unstack_timeseries(stacked_ts)
+ unstacked_data.iloc[:, 0] = data.iloc[:, 0].values
+
+ stacked_data = stack_timeseries(unstacked_data)
+ stacked_ts.loc[:, "series"] = stacked_data["series"].values
+ return stacked_ts
+
+
+def map_over_var_name_ts(path_ts, which, path_ts_updated):
+ """
+ Map and update time series data in directory 'path_ts' based on var_name.
+ Save the updated time series to the directory 'path_ts_updated'.
+ Delete the origin time series (empty_ts_...) in 'path_ts' directory.
+
+ Parameters
+ ----------
+ path_ts : str
+ The path to the original time series file.
+ which : str
+ Specifies the type of data value to map and update.
+ path_ts_updated : str
+ The path to save the updated time series file.
+
+ Returns
+ -------
+ None
+
+ """
+ # Create empty Dataframe for the updated time series
+ updated_ts_df = pd.DataFrame(columns=HEADER_B3_TS)
+
+ # Load set of time series
+ ts_set = load_b3_timeseries(path_ts)
+
+ # Loop over rows (set) of ts
+ for index, row in ts_set.iterrows():
+ check_if_ts_name_in_maps_ts(row["var_name"], "map_ts.yml")
+ datasets_file_name = get_datasets_data_name(row["var_name"], which)
+ datasets_file_path = get_abs_dataset_path(
+ "datasets", datasets_file_name.get("dataset"), data_dir=True
+ ) / datasets_file_name.get("file")
+
+ check_file_exists(
+ datasets_file_path, datasets_file_name, row, "map_ts.yml"
+ )
+
+ df_new_data_ts = pd.read_csv(datasets_file_path, usecols=[1], sep=",")
+
+ ts_updated = write_ts_data(
+ ts_set.iloc[[index], :].copy(), df_new_data_ts
+ )
+
+ updated_ts_df = pd.concat(
+ [updated_ts_df, ts_updated], ignore_index=False
+ )
+
+ updated_ts_df.index.name = ts_updated.index.name
+
+ save_df(updated_ts_df, path_ts_updated)
+
+ os.remove(path_ts)
+ # Print user info
+ logger.info(f"The file {path_ts} has been deleted.")
+
+
+if __name__ == "__main__":
+ path_empty_ts_efficiencies = sys.argv[1]
+ path_empty_ts_feedin = sys.argv[2]
+ path_empty_ts_load = sys.argv[3]
+
+ path_ts_efficiencies = sys.argv[4]
+ path_ts_feedin = sys.argv[5]
+ path_ts_load = sys.argv[6]
+
+ logger = add_snake_logger("data_processing")
+
+ map_over_var_name_ts(
+ path_empty_ts_efficiencies, "efficiencies", path_ts_efficiencies
+ )
+ map_over_var_name_ts(path_empty_ts_feedin, "feedin", path_ts_feedin)
+ map_over_var_name_ts(path_empty_ts_load, "load", path_ts_load)
diff --git a/digipipe/esys/snakemake_rules/build_datapackage.smk b/digipipe/esys/snakemake_rules/build_datapackage.smk
new file mode 100644
index 00000000..7eddb444
--- /dev/null
+++ b/digipipe/esys/snakemake_rules/build_datapackage.smk
@@ -0,0 +1,24 @@
+from digipipe.esys.esys.config.esys_conf import load_yaml
+
+def get_paths_scenario_input(wildcards):
+ scenario_specs = load_yaml(f"esys/scenarios/{wildcards.scenario}.yml")
+ paths_scenario_inputs = list()
+ for key in ["paths_scalars", "paths_timeseries"]:
+ paths = scenario_specs[key]
+ if isinstance(paths, list):
+ paths_scenario_inputs.extend(paths)
+ elif isinstance(paths, str):
+ paths_scenario_inputs.append(paths)
+ return paths_scenario_inputs
+
+rule build_datapackage:
+ input:
+ get_paths_scenario_input,
+ scenario="esys/scenarios/{scenario}.yml"
+ output: directory("store/appdata/esys/{scenario}/preprocessed")
+ params:
+ logfile="store/appdata/esys/{scenario}/{scenario}.log"
+ wildcard_constraints:
+ # Do not use this rule for the examples. Use prepare_example instead
+ scenario=r"(?!example_).*"
+ shell: "python esys/scripts/build_datapackage.py {input.scenario} {output} {params.logfile}"
diff --git a/digipipe/esys/snakemake_rules/create_empty_scalars.smk b/digipipe/esys/snakemake_rules/create_empty_scalars.smk
new file mode 100644
index 00000000..edfe8755
--- /dev/null
+++ b/digipipe/esys/snakemake_rules/create_empty_scalars.smk
@@ -0,0 +1,7 @@
+rule create_empty_scalars:
+ """
+ Create empty scalar files
+ """
+ input: "esys/scenarios/"
+ output: "store/datasets/esys_raw/data/scalars/empty_scalars.csv"
+ shell: "python esys/scripts/create_empty_scalars.py {input} {output}"
diff --git a/digipipe/esys/snakemake_rules/create_empty_ts.smk b/digipipe/esys/snakemake_rules/create_empty_ts.smk
new file mode 100644
index 00000000..6ba81936
--- /dev/null
+++ b/digipipe/esys/snakemake_rules/create_empty_ts.smk
@@ -0,0 +1,11 @@
+rule create_empty_ts:
+ """
+ Create empty timeseries files
+ """
+ input: "esys/scenarios/"
+ output:
+ "store/datasets/esys_raw/data/time_series/empty_ts_load.csv",
+ "store/datasets/esys_raw/data/time_series/empty_ts_feedin.csv",
+ "store/datasets/esys_raw/data/time_series/empty_ts_efficiencies.csv"
+
+ shell: "python esys/scripts/create_empty_ts.py {input} {output[0]} {output[1]} {output[2]}"
diff --git a/digipipe/esys/snakemake_rules/optimization.smk b/digipipe/esys/snakemake_rules/optimization.smk
new file mode 100644
index 00000000..7b2af8f2
--- /dev/null
+++ b/digipipe/esys/snakemake_rules/optimization.smk
@@ -0,0 +1,6 @@
+rule optimize:
+ input: "store/appdata/esys/{scenario}/preprocessed"
+ output: directory("store/appdata/esys/{scenario}/optimized/")
+ params:
+ logfile="store/appdata/esys/{scenario}/{scenario}.log"
+ shell: "python esys/scripts/optimize.py {input} {output} {params.logfile}"
diff --git a/digipipe/esys/snakemake_rules/postprocessing.smk b/digipipe/esys/snakemake_rules/postprocessing.smk
new file mode 100644
index 00000000..d2847aed
--- /dev/null
+++ b/digipipe/esys/snakemake_rules/postprocessing.smk
@@ -0,0 +1,6 @@
+rule postprocess:
+ input: "store/appdata/esys/{scenario}/optimized"
+ output: directory("store/appdata/esys/{scenario}/postprocessed/")
+ params:
+ logfile="store/appdata/esys/{scenario}/{scenario}.log"
+ shell: "python esys/scripts/postprocess.py {input} {wildcards.scenario} {output} {params.logfile}"
diff --git a/digipipe/esys/snakemake_rules/prepare_resource.smk b/digipipe/esys/snakemake_rules/prepare_resource.smk
new file mode 100644
index 00000000..c578faa8
--- /dev/null
+++ b/digipipe/esys/snakemake_rules/prepare_resource.smk
@@ -0,0 +1,5 @@
+rule prepare_scalars:
+ input:
+ raw_scalars="store/datasets/esys_raw/data/scalars/costs_efficiencies.csv",
+ output: "store/appdata/esys/_resources/scal_costs_efficiencies.csv"
+ shell: "python esys/scripts/prepare_scalars.py {input.raw_scalars} {output}"
diff --git a/digipipe/esys/snakemake_rules/write_costs_efficiencies.smk b/digipipe/esys/snakemake_rules/write_costs_efficiencies.smk
new file mode 100644
index 00000000..ac379fcd
--- /dev/null
+++ b/digipipe/esys/snakemake_rules/write_costs_efficiencies.smk
@@ -0,0 +1,9 @@
+rule write_costs_efficiencies:
+ """
+ Write costs and efficiencies from raw data to esys datasets
+ """
+ input:
+ "store/datasets/esys_raw/data/scalars/default_costs_efficiencies.csv",
+ "store/raw/technology_data/data/raw_costs_efficiencies.csv",
+ output: "store/datasets/esys_raw/data/scalars/costs_efficiencies.csv"
+ shell: "python esys/scripts/write_costs_efficiencies.py {input} {output}"
diff --git a/digipipe/esys/snakemake_rules/write_default_scalars.smk b/digipipe/esys/snakemake_rules/write_default_scalars.smk
new file mode 100644
index 00000000..d0992764
--- /dev/null
+++ b/digipipe/esys/snakemake_rules/write_default_scalars.smk
@@ -0,0 +1,9 @@
+rule write_default_scalars:
+ """
+ Write default values in scalar data files
+ """
+ input: "store/datasets/esys_raw/data/scalars/empty_scalars.csv"
+ output:
+ "store/datasets/esys_raw/data/scalars/default_scalars.csv",
+ "store/datasets/esys_raw/data/scalars/default_costs_efficiencies.csv",
+ shell: "python esys/scripts/write_default_scalars.py {input} {output}"
diff --git a/digipipe/esys/snakemake_rules/write_ts.smk b/digipipe/esys/snakemake_rules/write_ts.smk
new file mode 100644
index 00000000..ab94168a
--- /dev/null
+++ b/digipipe/esys/snakemake_rules/write_ts.smk
@@ -0,0 +1,10 @@
+rule write_ts:
+ input:
+ "store/datasets/esys_raw/data/time_series/empty_ts_efficiencies.csv",
+ "store/datasets/esys_raw/data/time_series/empty_ts_feedin.csv",
+ "store/datasets/esys_raw/data/time_series/empty_ts_load.csv",
+ output:
+ "store/datasets/esys_raw/data/time_series/ts_efficiencies.csv",
+ "store/datasets/esys_raw/data/time_series/ts_feedin.csv",
+ "store/datasets/esys_raw/data/time_series/ts_load.csv",
+ shell: "python esys/scripts/write_ts.py {input} {output}"
diff --git a/digipipe/scenarios/SCENARIOS.md b/digipipe/scenarios/SCENARIOS.md
new file mode 100644
index 00000000..1de8f146
--- /dev/null
+++ b/digipipe/scenarios/SCENARIOS.md
@@ -0,0 +1,3 @@
+# Scenarios
+
+**TBD**
diff --git a/digipipe/store/1_converted/.gitkeep b/digipipe/scripts/__init__.py
similarity index 100%
rename from digipipe/store/1_converted/.gitkeep
rename to digipipe/scripts/__init__.py
diff --git a/digipipe/scripts/config.py b/digipipe/scripts/config.py
new file mode 100644
index 00000000..6a02d241
--- /dev/null
+++ b/digipipe/scripts/config.py
@@ -0,0 +1,78 @@
+"""
+Helper functions for configs
+"""
+
+import os
+from pathlib import Path
+
+import yaml
+
+from digipipe.store.utils import get_abs_store_root_path
+
+
+def read_config(file: Path) -> dict:
+ """Reads a yml config and returns as dict
+
+ Parameters
+ ----------
+ file : str
+ Full path to config file to read
+
+ Returns
+ -------
+ dict
+ Config dict
+ """
+ with open(file, "r") as cfg_file:
+ try:
+ cfg = yaml.safe_load(cfg_file) or {}
+ except yaml.YAMLError as exc:
+ print(f"Error while reading config file: {file}")
+ print(exc)
+ return cfg
+
+
+def load_dataset_configs() -> dict:
+ """Load datasets' yml configs and merge them using the directory tree for
+ naming.
+
+ Parameters
+ ----------
+ config_files : list of str
+ List of paths to config files
+ Returns
+ -------
+ dict
+ Config dict of format:
+ {"store":
+ {"category":
+ {"": {}, ...}
+ }
+ }
+ """
+
+ def search_store_configs() -> list:
+ """Search for configs (*.yml) in data store, exclude templates.
+
+ Returns
+ -------
+ list
+ Paths to config files
+ """
+ cfg_files = list()
+ for root, dirs, files in os.walk(get_abs_store_root_path()):
+ for file in files:
+ if (file == "config.yml") and ".TEMPLATE" not in str(root):
+ cfg_files.append(Path(os.path.join(root, file)))
+ return cfg_files
+
+ merged_cfg = dict()
+ for file in search_store_configs():
+ path = Path(file).resolve()
+ section = path.parent.parent.name
+ subsection = path.parent.name
+ cfg = read_config(file)
+ if merged_cfg.get(section, None) is None:
+ merged_cfg[section] = {}
+ merged_cfg[section][subsection] = cfg
+ return {"store": merged_cfg}
diff --git a/digipipe/scripts/data_io.py b/digipipe/scripts/data_io.py
new file mode 100644
index 00000000..ec13380e
--- /dev/null
+++ b/digipipe/scripts/data_io.py
@@ -0,0 +1,120 @@
+import json
+import os
+import shutil
+import zipfile
+from pathlib import Path
+
+import requests
+
+
+def load_json(file_path: Path) -> dict:
+ with open(file_path, "r") as f:
+ return json.load(f)
+
+
+def download_file(url: str, dest_file: str) -> None:
+ """Downloads file from url to specified destination.
+
+ Parameters
+ ----------
+ url : str
+ The URL string of the file to be downloaded.
+ dest_file : str
+ The destination path/filename of the file to be downloaded.
+ """
+ print(f"Downloading from {url}")
+ with requests.get(url, stream=True, timeout=10) as r:
+ r.raise_for_status()
+ with open(dest_file, "wb") as f:
+ for i, block in enumerate(r.iter_content(chunk_size=1024)):
+ f.write(block)
+ downloaded_size = i * 1024 / 1e6
+ print(f"{downloaded_size:.2f} MB...", end="\r")
+ print(f"Download completed to {dest_file}")
+
+
+def extract_zipfile(zip_path: str, dest_path: str) -> None:
+ """Extracts zipfile to destination path
+
+ Parameters
+ ----------
+ zip_path : str
+ Path to the zipfile to be extracted
+ dest_path : str
+ Destination path where the contents of zipfile are to be extracted
+ """
+ with zipfile.ZipFile(zip_path, "r") as zip_ref:
+ zip_ref.extractall(dest_path)
+
+
+def copy_files(src_path: str, dest_path: str) -> None:
+ """Copies files from source to destination directory
+
+ Parameters
+ ----------
+ src_path : str
+ Source path of directory to copy from
+ dest_path : str
+ Destination path of directory to copy to
+ """
+ # Get a list of directories within source directory
+ dir_list = [
+ d
+ for d in os.listdir(src_path)
+ if os.path.isdir(os.path.join(src_path, d))
+ ]
+ # Loop through each directory and copy files within `data` directory
+ for d in dir_list:
+ src_dir = os.path.join(src_path, d, "data")
+ dst_dir = os.path.join(dest_path, d, "data")
+ # Check if destination directory for data already exists
+ if os.path.exists(dst_dir):
+ files_to_copy = []
+ for file in os.listdir(src_dir):
+ src_file = os.path.join(src_dir, file)
+ dst_file = os.path.join(dst_dir, file)
+ if os.path.isfile(dst_file):
+ if file == ".gitkeep":
+ continue
+ print(f"\n'{file}' already exists in '{d}/data'.")
+ overwrite_file = input("Do you want to update it? (y/n) ")
+ while overwrite_file.lower() not in ["y", "n"]:
+ overwrite_file = input(
+ """Invalid input. Enter 'y' or 'n': """
+ )
+ if overwrite_file.lower() == "y":
+ shutil.copy(src_file, dst_file)
+ print(f"'{file}' updated.")
+ else:
+ continue
+ else:
+ files_to_copy.append(file)
+
+ for file in files_to_copy:
+ src_file = os.path.join(src_dir, file)
+ dst_file = os.path.join(dst_dir, file)
+ shutil.copy(src_file, dst_file)
+ print(f"\n'{file}' added to '{d}/data'.")
+ # If directory doesn't exist, copy it entirely
+ else:
+ shutil.copytree(src_dir, dst_dir)
+ print(f"\nAdded '{d}' to 'store/raw'.")
+
+
+def clean_folder(folder_path: str) -> None:
+ """Cleans up a given folder by removing all files and directories within it
+ (except for .gitkeep)
+
+ Parameters
+ ----------
+ folder_path : str
+ Path to the folder that should be cleaned
+ """
+ # Clean temp folder
+ folder_content = os.listdir(folder_path)
+ for item in folder_content:
+ item_path = os.path.join(folder_path, item)
+ if os.path.isfile(item_path) and item != ".gitkeep":
+ os.remove(item_path)
+ elif os.path.isdir(item_path):
+ shutil.rmtree(item_path)
diff --git a/digipipe/store/2_preprocessed/.gitkeep b/digipipe/scripts/datasets/__init__.py
similarity index 100%
rename from digipipe/store/2_preprocessed/.gitkeep
rename to digipipe/scripts/datasets/__init__.py
diff --git a/digipipe/scripts/datasets/demand.py b/digipipe/scripts/datasets/demand.py
new file mode 100644
index 00000000..8454ee64
--- /dev/null
+++ b/digipipe/scripts/datasets/demand.py
@@ -0,0 +1,247 @@
+"""Shared functions for processing electricity and heat demand datasets"""
+
+from pathlib import Path
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+
+
+def normalize_filter_timeseries(
+ infile: Path, outfile: Path, region_nuts: list = None
+) -> None:
+ """Extract timeseries for specific districts, merge and normalize them to
+ 1 (MWh)
+
+ Parameters
+ ----------
+ infile : pathlib.Path
+ Path to timeseries CSV with NUTS 3 codes in columns and
+ timesteps in rows (expects 15min ot 1h resolution)
+ outfile : pathlib.Path
+ Path to CSV outfile - aggregated and normalized timeseries
+ (1h resolution)
+ region_nuts : list of str
+ List of NUTS 3 codes whose timeseries are to be used
+
+ Returns
+ -------
+ None
+ """
+ # Get timeseries, create timeindex, filter region, resample to 1h
+ timeseries = pd.read_csv(infile)
+ timeseries = timeseries[region_nuts]
+
+ if len(timeseries) == 8760:
+ timeseries.index = pd.DatetimeIndex(
+ pd.date_range(
+ start="2017-01-01 00:00:00",
+ end="2017-12-31 23:45:00",
+ freq="1H",
+ )
+ )
+ elif len(timeseries) == 35040:
+ timeseries.index = pd.DatetimeIndex(
+ pd.date_range(
+ start="2017-01-01 00:00:00",
+ end="2017-12-31 23:45:00",
+ freq="15Min",
+ )
+ )
+ timeseries = timeseries.resample("H").sum()
+ else:
+ raise ValueError("Invalid number of rows in timeseries!")
+
+ # Average SLP timeseries and normalize to 1 MWh
+ timeseries = timeseries.sum(axis=1)
+ timeseries = timeseries.div(timeseries.sum()).reset_index(drop=True)
+ timeseries.name = "demand_norm"
+
+ # Check result
+ np.testing.assert_almost_equal(
+ timeseries.sum(),
+ 1,
+ err_msg="Sum of normalized timeseries is not 1!",
+ )
+
+ # Write
+ timeseries.to_csv(outfile)
+
+
+def merge_demand_multiple_years(
+ infiles: list,
+ outfile: Path,
+) -> None:
+ """Merge demand from different years into one
+
+ Parameters
+ ----------
+ infiles : list of Path
+ CSV with each holding demand per municipality for one year
+ outfile : pathlib.Path
+ CSV with demand per municipality for all years
+
+ Returns
+ -------
+ None
+ """
+ demand = pd.concat(
+ [pd.read_csv(f, index_col="municipality_id") for f in infiles],
+ axis=1,
+ )
+ demand.to_csv(outfile)
+
+
+def disaggregate_demand_to_municipality(
+ demand_districts: pd.DataFrame,
+ muns: gpd.GeoDataFrame,
+ districts: gpd.GeoDataFrame,
+ disagg_data: pd.DataFrame,
+ disagg_data_col: str,
+) -> pd.DataFrame:
+ """Disaggregates NUTS 3 consumption to municipalities by linear scaling
+ using factors from `disagg_data`.
+
+ Parameters
+ ----------
+ demand_districts : pd.DataFrame
+ Annual demand per district (NUTS 3 level)
+ muns : gpd.GeoDataFrame
+ Municipalities
+ districts : gpd.GeoDataFrame
+ Federal districts
+ disagg_data : pd.DataFrame
+ DF that contains municipal data to scale demand by
+ disagg_data_col : str
+ Name of DF column used for scaling
+
+ Returns
+ -------
+ pd.DataFrame
+ Annual demand per municipality
+ """
+ # Get muns and their NUTS3 code
+ muns = muns.merge(
+ districts[["id", "nuts"]].rename(columns={"id": "district_id"}),
+ left_on="district_id",
+ right_on="district_id",
+ )
+ disagg_data = (
+ disagg_data.reset_index()
+ .merge(muns[["id", "nuts"]], left_on="municipality_id", right_on="id")
+ .drop(columns=["id"])
+ )
+
+ # Calc relative share within each district
+ disagg_data = disagg_data.assign(
+ pop_share=(
+ disagg_data[disagg_data_col]
+ / disagg_data.groupby("nuts")[disagg_data_col].transform("sum")
+ )
+ ).set_index("nuts")
+
+ # Filter demand dataset
+ demand_districts = demand_districts.loc[districts.nuts.to_list()]
+
+ # Merge demand and calc value per municipality
+ demand = disagg_data.merge(
+ demand_districts,
+ left_index=True,
+ right_index=True,
+ )
+ demand[disagg_data_col] = demand_districts.demand_districts.mul(
+ demand.pop_share
+ )
+ demand = demand[["municipality_id", disagg_data_col]].set_index(
+ "municipality_id"
+ )
+
+ # Check result
+ np.testing.assert_almost_equal(
+ demand.sum().sum(),
+ demand_districts.sum().sum(),
+ err_msg="Sum of disaggregated values does not match total value!",
+ )
+
+ return demand
+
+
+def demand_prognosis(
+ demand_future_T45: Path,
+ demand_future_TN: Path,
+ demand_region: pd.DataFrame,
+ year_base: int,
+ year_target: int,
+ scale_by: str,
+ carrier: str = None,
+) -> pd.DataFrame:
+ """Create demand prognosis for target year per municipality by scaling with
+ country data (by total demand or carrier)
+
+ Parameters
+ ----------
+ demand_future_T45 : pathlib.Path
+ Path to total demand per carrier in Germany (BMWK T45 scenario)
+ demand_future_TN : pathlib.Path
+ Path to total demand per carrier in Germany (BMWK TN scenario)
+ demand_region : pd.DataFrame
+ Annual demand per municipality (today)
+ year_base : int
+ Base year to use for calculation of reduction
+ year_target : int
+ Target year for demand
+ scale_by : str
+ One of ["total", "carrier"], scale future demand by total energy or
+ specific carrier given by `carrier`
+ carrier : str
+ Carrier
+
+
+ Returns
+ -------
+ pd.DataFrame
+ Annual demand per municipality (prognosis)
+ """
+ if scale_by not in ["total", "carrier"]:
+ raise ValueError("scale_by must be one of ['total', 'carrier']")
+ if scale_by == "carrier":
+ if carrier is None:
+ raise ValueError("carrier must be provided when scale_by='carrier'")
+
+ # Get data from future scenarios
+ demand_future_T45 = pd.read_csv(demand_future_T45).set_index(
+ ["year", "carrier"]
+ )
+ demand_future_TN = pd.read_csv(demand_future_TN)
+
+ # Interpolate for base year
+ demand_future_TN = demand_future_TN.set_index(["year", "carrier"]).append(
+ pd.DataFrame(
+ index=pd.MultiIndex.from_product(
+ [[year_base], demand_future_TN.carrier.unique(), [np.nan]],
+ names=demand_future_TN.columns,
+ )
+ )
+ )
+ demand_future_TN.sort_index(inplace=True)
+ demand_future_TN = demand_future_TN.unstack(level=1).interpolate().stack()
+
+ # Scale by total or carrier demand
+ if scale_by == "total":
+ # Total reduction factor
+ reduction_factor = (
+ demand_future_T45.loc[year_target].demand.sum()
+ / demand_future_TN.loc[year_base].demand.sum()
+ )
+ elif scale_by == "carrier":
+ # Reduction factor for carrier
+ reduction_factor = (
+ demand_future_T45.loc[year_target, carrier].demand
+ / demand_future_TN.loc[year_base, carrier].demand
+ )
+ print(
+ f"Total demand in {year_target}: {round(reduction_factor, 2)} of "
+ f"{year_base}'s demand."
+ )
+
+ return demand_region * reduction_factor
diff --git a/digipipe/scripts/datasets/mastr.py b/digipipe/scripts/datasets/mastr.py
new file mode 100644
index 00000000..3db09b0b
--- /dev/null
+++ b/digipipe/scripts/datasets/mastr.py
@@ -0,0 +1,483 @@
+"""Shared functions for processing data from MaStR dataset"""
+
+from typing import Tuple, Union
+
+import geopandas as gpd
+import pandas as pd
+from geopy.exc import GeocoderUnavailable
+from geopy.extra.rate_limiter import RateLimiter
+from geopy.geocoders import Nominatim
+
+
+def cleanse(
+ units: Union[pd.DataFrame, gpd.GeoDataFrame]
+) -> Union[pd.DataFrame, gpd.GeoDataFrame]:
+ """Do some basic cleansing of MaStR unit data.
+
+ This involves:
+ * country
+
+ Parameters
+ ----------
+ units : pd.DataFrame or gpd.GeoDataFrame
+ Units from MaStR
+
+ Returns
+ -------
+ pd.DataFrame or gpd.GeoDataFrame
+ Filtered units
+ """
+ units = units.loc[(units.country == "Deutschland")]
+
+ # Drop unnecessary columns
+ units.drop(
+ columns=[
+ "country",
+ ],
+ inplace=True,
+ )
+
+ return units
+
+
+def apply_manual_corrections(
+ units_df: pd.DataFrame,
+ units_correction_df: pd.DataFrame,
+) -> pd.DataFrame:
+ """Correct units using manual correction dataset
+
+ Parameters
+ ----------
+ units_df : pd.DataFrame
+ Units from MaStR
+ units_correction_df : pd.DataFrame
+ Correction data
+ Returns
+ -------
+ pd.DataFrame
+ Corrected units
+ """
+ for attr in units_correction_df.wrong_attr.unique():
+ # Correct site type (roof- or ground-mounted)
+ if attr == "site_type":
+ units_correction_site_df = units_correction_df.copy().loc[
+ units_correction_df.wrong_attr == "site_type"
+ ][["correction"]]
+ units_df.loc[
+ units_correction_site_df.index, "Lage"
+ ] = units_correction_site_df["correction"]
+ print(
+ f"Applied {len(units_correction_site_df)} "
+ f"corrections for column: {attr}"
+ )
+ # Correct geometry
+ elif attr == "geometry":
+ units_correction_geom_df = units_correction_df.copy()
+ units_correction_geom_df[["x", "y"]] = (
+ units_correction_geom_df.loc[
+ (units_correction_geom_df.wrong_attr == "geometry")
+ & (units_correction_geom_df.correction != "None")
+ ]
+ .correction.str.split(",", expand=True)
+ .astype(float)
+ )
+ units_correction_geom_df = gpd.GeoDataFrame(
+ units_correction_geom_df,
+ geometry=gpd.points_from_xy(
+ units_correction_geom_df.x,
+ units_correction_geom_df.y,
+ crs=3035,
+ ),
+ crs=3035,
+ ).to_crs(4326)
+ units_correction_geom_df = units_correction_geom_df.loc[
+ ~units_correction_geom_df.geometry.is_empty
+ ]
+ units_correction_geom_df = units_correction_geom_df.assign(
+ lon=units_correction_geom_df.geometry.x,
+ lat=units_correction_geom_df.geometry.y,
+ )
+ units_df.loc[
+ units_correction_geom_df.index, ["Laengengrad", "Breitengrad"]
+ ] = units_correction_geom_df[["lon", "lat"]]
+ print(
+ f"Applied {len(units_correction_geom_df)} "
+ f"corrections for column: {attr}"
+ )
+ # If attribute does not exist
+ else:
+ raise NotImplementedError(
+ f"Correction of PV roof for attribute '{attr}' is not supported"
+ )
+ return units_df.reset_index()
+
+
+def add_voltage_level(
+ units_df: pd.DataFrame,
+ locations_path: str,
+ gridconn_path: str,
+ drop_location_id: bool = True,
+) -> pd.DataFrame:
+ """Add voltage level to units from MaStR using locations and grid
+ connection points.
+
+ Parameters
+ ----------
+ units_df : pd.DataFrame
+ Units from MaStR
+ locations_path : str
+ Path to MaStR locations file
+ gridconn_path : str
+ Path to MaStR grid connection points file
+ drop_location_id : bool
+ Drop location id in the end
+
+ Returns
+ -------
+ pd.DataFrame
+ Units with column `voltage_level`
+ """
+ # Get locations and grid connection point and merge both
+ locations = pd.read_csv(
+ locations_path,
+ usecols=["MastrNummer", "Netzanschlusspunkte"],
+ ).rename(columns={"MastrNummer": "mastr_location_id2"})
+ gridconn = pd.read_csv(
+ gridconn_path,
+ usecols=["NetzanschlusspunktMastrNummer", "Spannungsebene"],
+ )
+ locations = (
+ locations.merge(
+ gridconn,
+ left_on="Netzanschlusspunkte",
+ right_on="NetzanschlusspunktMastrNummer",
+ how="left",
+ )
+ .drop_duplicates()
+ .rename(columns={"Spannungsebene": "voltage_level"})
+ )
+
+ # Add voltage level to units
+ units_df = units_df.reset_index().merge(
+ locations[["mastr_location_id2", "voltage_level"]],
+ left_on="mastr_location_id",
+ right_on="mastr_location_id2",
+ how="left",
+ )
+
+ # Drop unnecessary columns
+ cols = (
+ ["mastr_location_id", "mastr_location_id2"]
+ if drop_location_id
+ else ["mastr_location_id2"]
+ )
+ units_df.drop(
+ columns=cols,
+ inplace=True,
+ )
+
+ return units_df
+
+
+def add_geometry(
+ units_df: pd.DataFrame,
+ drop_units_wo_coords: bool = True,
+) -> gpd.GeoDataFrame:
+ """
+ Add `geometry` column to MaStR unit data using `lat` and `lon` values.
+ Columns `lat` and `lon` are dropped
+
+ Parameters
+ ----------
+ units_df : pd.DataFrame
+ Units with columns `lat` and `lon` in CRS WGS84 (EPSG:4326)
+ drop_units_wo_coords : bool
+ Drop units which do not have valid lat and lon values.
+ Note: coordinates in the MaStR are only provided for plants>30 kW.
+
+ Returns
+ -------
+ gpd.GeoDataFrame
+ Units with geometry in CRS LAEA Europe (EPSG:3035)
+ """
+ # Drop units without coords idf requested
+ if drop_units_wo_coords:
+ units_count_orig = len(units_df)
+ units_df = units_df.loc[(~units_df.lon.isna() & ~units_df.lat.isna())]
+ print(
+ f"{units_count_orig-len(units_df)} units have no or invalid "
+ f"coordinates. Their locations will be geocoded."
+ )
+
+ units_gdf = gpd.GeoDataFrame(
+ units_df,
+ geometry=gpd.points_from_xy(units_df["lon"], units_df["lat"], crs=4326),
+ crs=4326,
+ ).to_crs(3035)
+
+ # Drop unnecessary columns
+ units_gdf.drop(columns=["lon", "lat"], inplace=True)
+
+ return units_gdf
+
+
+def geocode(
+ units_df: pd.DataFrame,
+ user_agent: str = "geocoder",
+ interval: int = 1,
+ target_crs: str = "EPSG:3035",
+) -> gpd.GeoDataFrame:
+ """
+ Geocode locations from MaStR unit table using zip code and city.
+
+ Parameters
+ ----------
+ units_df : pd.DataFrame
+ Units from MaStR. Must contain the following columns:
+ * zip_code (str)
+ * city (str)
+ user_agent : str
+ Some app name. Defaults to "geocoder"
+ interval : int
+ Delay in seconds to use between requests to Nominatim.
+ A minimum of 1 is advised (default)
+ target_crs : str
+ CRS the data should be reprojected to. Defaults to EPSG:3035.
+
+ Returns
+ -------
+ gpd.GeoDataFrame
+ Units with geometry
+ """
+
+ def geocoder(
+ user_agent: str,
+ interval: int,
+ ) -> RateLimiter:
+ """Setup Nominatim geocoding class.
+
+ Parameters
+ -----------
+ user_agent : str
+ Some app name.
+ interval : int
+ Delay in seconds to use between requests to Nominatim.
+ A minimum of 1 is advised.
+
+ Returns
+ -------
+ geopy.extra.rate_limiter.RateLimiter
+ Nominatim RateLimiter geocoding class to use for geocoding.
+ """
+ try:
+ locator = Nominatim(user_agent=user_agent)
+ return RateLimiter(
+ locator.geocode,
+ min_delay_seconds=interval,
+ )
+ except GeocoderUnavailable:
+ print("Geocoder unavailable, aborting geocoding!")
+ raise
+
+ # Define geocoder
+ ratelimiter = geocoder(
+ user_agent,
+ interval,
+ )
+
+ # Merge zip code and city and get unique values
+ units_df = units_df.assign(
+ zip_and_city=units_df.zip_code.astype(str) + " " + units_df.city,
+ )
+ unique_locations = pd.DataFrame(
+ data=units_df.zip_and_city.unique(),
+ columns=["zip_and_city"],
+ )
+ # Geocode unique locations!
+ print(
+ f"Geocoding {len(unique_locations)} unique locations, this will take "
+ f"about {round(len(unique_locations) * interval / 60, 1)} min..."
+ )
+ unique_locations = unique_locations.assign(
+ location=unique_locations.zip_and_city.apply(ratelimiter)
+ )
+ unique_locations = unique_locations.assign(
+ point=unique_locations.location.apply(
+ lambda loc: tuple(loc.point) if loc else None
+ )
+ )
+ unique_locations[["latitude", "longitude", "altitude"]] = pd.DataFrame(
+ unique_locations.point.tolist(), index=unique_locations.index
+ )
+ unique_locations = gpd.GeoDataFrame(
+ unique_locations,
+ geometry=gpd.points_from_xy(
+ unique_locations.longitude, unique_locations.latitude
+ ),
+ crs="EPSG:4326",
+ )
+ # Merge locations back in units
+ units_gdf = gpd.GeoDataFrame(
+ units_df.merge(
+ unique_locations[["zip_and_city", "geometry"]],
+ on="zip_and_city",
+ ).drop(columns=["zip_and_city"])
+ ).to_crs(target_crs)
+
+ return units_gdf
+
+
+def geocode_units_wo_geometry(
+ units_df: pd.DataFrame,
+ columns_agg_functions: dict,
+ target_crs: str = "EPSG:3035",
+) -> Tuple[gpd.GeoDataFrame, gpd.GeoDataFrame]:
+ """
+ Add locations to units without coordinates by geocoding. The units are
+ returned with approximated locations (1 unit per row) as well as grouped by
+ approximated location (1 dataset with >=1 units per row) with aggregated
+ attributes as given by `columns_agg_functions`.
+
+ Parameters
+ ----------
+ units_df : pd.DataFrame
+ Units without geometry. Must have columns "zip_code" and "city" as well
+ as the columns specified in `columns_agg_functions`.
+ Note: If existent, geometry column will be overwritten.
+ columns_agg_functions : dict
+ Defines how columns shall be aggregated. Format as in Pandas' .agg()
+ function.
+ Example:
+ {
+ "capacity_net": ("capacity_net", "sum"),
+ "unit_count": ("capacity_net", "count"),
+ "capacity_gross": ("capacity_gross", "sum")
+ }
+ In this example, the sum of `capacity_net` and number of units is
+ calculated and named to "capacity_net" and "unit_count", respectively.
+ Also, the sum of `capacity_gross` is calculated and the name is
+ retained.
+ target_crs : str
+ CRS the data should be reprojected to. Defaults to EPSG:3035
+
+ Returns
+ -------
+ gpd.GeoDataFrame
+ Units with approximated location (1 unit per row).
+ gpd.GeoDataFrame
+ Units grouped by approximated location (1 dataset with >=1 units per
+ row) with aggregated attributes as given by `columns_agg_functions`.
+ """
+
+ def aggregate_units_wo_geometry(
+ units_gdf: gpd.GeoDataFrame,
+ ) -> gpd.GeoDataFrame:
+ """Aggregate units by approximated position
+
+ Parameters
+ ----------
+ units_gdf : gpd.GeoDataFrame
+ Units
+ Returns
+ -------
+ gpd.GeoDataFrame
+ Units aggregated by position
+ """
+
+ # Aggregate units with approximated position
+ units_gdf["lon"] = units_gdf.geometry.x
+ units_gdf["lat"] = units_gdf.geometry.y
+
+ grouping_columns = ["zip_code", "city", "lat", "lon"]
+ units_agg_gdf = (
+ units_gdf[grouping_columns + columns_agg_names]
+ .groupby(grouping_columns, as_index=False)
+ .agg(**columns_agg_functions)
+ )
+
+ # Create geometry and select columns
+ units_agg_gdf = gpd.GeoDataFrame(
+ units_agg_gdf,
+ geometry=gpd.points_from_xy(units_agg_gdf.lon, units_agg_gdf.lat),
+ crs=target_crs,
+ )[
+ ["zip_code", "city"]
+ + list(columns_agg_functions.keys())
+ + ["geometry"]
+ ]
+ return units_agg_gdf.assign(
+ status="In Betrieb oder in Planung",
+ geometry_approximated=1,
+ )
+
+ # Check if all required columns are present
+ if not all(c in units_df.columns for c in ["zip_code", "city"]):
+ raise ValueError(
+ "Column zip_code or city not present, geocoding not possible."
+ )
+ columns_agg_names = list({c for c, _ in columns_agg_functions.values()})
+ if not all(c in units_df.columns for c in columns_agg_names):
+ raise ValueError(
+ "On or more columns requested in the aggregation functions dict "
+ "(columns_agg_functions) are not present, cannot proceed."
+ )
+
+ units_with_inferred_geom_gdf = geocode(units_df)
+ units_with_inferred_geom_gdf = units_with_inferred_geom_gdf.assign(
+ geometry_approximated=1,
+ )
+
+ units_with_inferred_geom_agg_gdf = aggregate_units_wo_geometry(
+ units_with_inferred_geom_gdf.copy()
+ )
+
+ return units_with_inferred_geom_gdf, units_with_inferred_geom_agg_gdf
+
+
+def create_stats_per_municipality(
+ units_df: pd.DataFrame,
+ muns: gpd.GeoDataFrame,
+ column: str,
+ only_operating_units: bool = True,
+) -> pd.DataFrame:
+ """
+ Create statistics on units per municipality for one column.
+ Municipalities with no data in `column` are set to 0.
+
+ Parameters
+ ----------
+ units_df : pd.DataFrame
+ Units
+ muns : gpd.GeoDataFrame
+ Municipalities
+ column : str
+ Column in units_df used for aggregation
+ only_operating_units : bool
+ Use only units which are operating, no planned or decommissioned.
+ Defaults to true.
+
+ Returns
+ -------
+ pd.DataFrame
+ Units, aggregated per municipality
+ """
+
+ if only_operating_units is True:
+ units_df = units_df.loc[units_df["status"] == "In Betrieb"]
+
+ units_df = units_df[["municipality_id", column]]
+ units_df = (
+ units_df.groupby("municipality_id")
+ .agg(
+ column=(column, "sum"),
+ count=(column, "count"),
+ )
+ .rename(columns={"column": column})
+ )
+
+ units_df = units_df.reindex(muns.id, fill_value=0).rename(
+ columns={"id": "municipality_id"}
+ )
+ units_df.index.name = "municipality_id"
+
+ return units_df
diff --git a/digipipe/scripts/geo.py b/digipipe/scripts/geo.py
new file mode 100644
index 00000000..ece33554
--- /dev/null
+++ b/digipipe/scripts/geo.py
@@ -0,0 +1,409 @@
+"""
+Helper functions for geodata processing
+"""
+
+import os
+from collections import OrderedDict
+from typing import Tuple, Union
+
+import fiona
+import geopandas as gpd
+import pandas as pd
+import rasterio as rio
+from rasterio.mask import mask
+from rasterstats import zonal_stats
+from shapely.geometry.multipolygon import MultiPolygon
+from shapely.ops import transform
+
+from digipipe.config import GLOBAL_CONFIG
+
+
+def read_schema_from_file(file: str) -> Tuple[str, OrderedDict]:
+ """Read a geo file and returns schema definition using fiona
+
+ Parameters
+ ----------
+ file : str
+ Full path to file to read schema from
+
+ Returns
+ -------
+ str
+ Schema of geometry
+ OrderedDict
+ Properties/Fields of dataset (str: str)
+ """
+ try:
+ with fiona.open(file) as f:
+ schema_in_geom = f.schema["geometry"]
+ schema_in_props = f.schema["properties"]
+ except:
+ f.close()
+ raise
+ return schema_in_geom, schema_in_props
+
+
+def file_is_empty(file: str) -> bool:
+ """Check if file holds no geodata (is empty)
+
+ Parameters
+ ----------
+ file : str
+ Full path to file to read
+
+ Returns
+ -------
+ bool
+ True if file is empty, False otherwise
+ """
+ with fiona.open(file) as coll:
+ return len(coll) == 0
+
+
+def convert_to_multipolygon(gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame:
+ """Convert geometry column to type MultiPolygon
+
+ Parameters
+ ----------
+ gdf : :pandas:`geopandas.GeoDataFrame`
+ Data to be aligned
+
+ Returns
+ -------
+ :pandas:`geopandas.GeoDataFrame`
+ """
+
+ def remove_z(row):
+ """
+ Remove z coordinate from Geometry, e.g. MultiPolygon (MULTIPOLYGON Z)
+ """
+ return transform(lambda x, y, z=None: (x, y), row)
+
+ gdf["geometry"] = [
+ MultiPolygon([feature]) if feature.geom_type == "Polygon" else feature
+ for feature in gdf["geometry"]
+ ]
+
+ gdf["geometry"] = gdf["geometry"].apply(remove_z)
+
+ return gdf
+
+
+def write_geofile(
+ gdf: gpd.GeoDataFrame,
+ file: str,
+ layer_name: str = None,
+ schema: dict = None,
+ driver: str = "GPKG",
+ encoding: str = "utf-8",
+) -> None:
+ """Write geodata to file
+
+ Parameters
+ ----------
+ gdf : :pandas:`geopandas.GeoDataFrame`
+ file : str
+ Target file
+ layer_name : str
+ Name of layer, usually same as file basename
+ schema : dict
+ Output schema with keys "geometry" and "properties"
+ driver : str
+ Geofile driver, default is Geopackage
+ encoding : str
+ Encoding
+ """
+ if layer_name is None:
+ layer_name = os.path.basename(file).split(".")[0]
+ if schema is None:
+ # TODO: Log warning
+ pass
+
+ # check if data contain multiple geometry types
+ if len(gdf.geometry.type.unique()) > 1:
+ types = gdf.geometry.type.unique()
+ raise ValueError(f"Data contain multiple geometry types: {types} !")
+
+ gdf.to_file(
+ file, layer=layer_name, schema=schema, driver=driver, encoding=encoding
+ )
+
+
+def rename_filter_attributes(
+ gdf: Union[pd.DataFrame, gpd.GeoDataFrame],
+ attrs_filter_by_values: dict = None,
+ attrs_mapping: dict = None,
+) -> Union[pd.DataFrame, gpd.GeoDataFrame]:
+ """Rename attributes and filter them by values
+
+ Note: Only attributes as given by `attrs_mapping` are kept!
+
+ Parameters
+ ----------
+ gdf : pd.DataFrame or gpd.GeoDataFrame
+ Geodata
+ attrs_filter_by_values : dict
+ Attributes whose values are to be filtered. Use attributes as dict
+ keys and desired values as dict values (values can be of type str,
+ int, float or list)
+ Example: {"GF": 4, "NUTS": ["DEE01", "DEE05", "DEE0E"]}
+ attrs_mapping : dict
+ Attributes to select and rename. Use original attributes' names as
+ dict keys and new names as values.
+
+ Returns
+ -------
+ pd.DataFrame or gpd.GeoDataFrame
+ Filtered geodata
+ """
+ # Filter by attribute values, if defined
+ if attrs_filter_by_values is not None:
+ list_vals = []
+ query = ""
+ for k, v in attrs_filter_by_values.items():
+ if isinstance(v, list):
+ list_vals.append(v)
+ query += f" & {k} in @list_vals[{len(list_vals)-1}]"
+ elif isinstance(v, str):
+ query += f" & {k}=='{v}'"
+ elif isinstance(v, (int, float)):
+ query += f" & {k}=={v}"
+ else:
+ raise ValueError(
+ "Data type in attribute filter is not supported!"
+ )
+ query = query[2:]
+ gdf = gdf.query(query)
+
+ # Extract and rename fields
+ if attrs_mapping is not None:
+ gdf = gdf.filter(attrs_mapping.keys())
+ gdf.rename(columns=attrs_mapping, inplace=True)
+
+ return gdf
+
+
+def reproject_simplify(
+ gdf: gpd.GeoDataFrame,
+ target_crs: str = GLOBAL_CONFIG["global"]["geodata"]["crs"].lower(),
+ min_size: float = None,
+ simplify_tol: float = None,
+ fix_geom: bool = False,
+ add_id_column: bool = False,
+) -> gpd.GeoDataFrame:
+ """General purpose function for processing of geodata
+
+ Parameters
+ ----------
+ gdf : gpd.GeoDataFrame
+ Geodata
+ target_crs : str
+ CRS the data should be reprojected to. Defaults to value from global
+ config.
+ min_size : float
+ Min. size of area to select (in sqm). Use None for no filtering
+ (default).
+ Raises exception if `target_crs` is not LAEA Europe (EPSG:3035).
+ simplify_tol : float
+ Threshold for simplification of geometries. Use None for no
+ simplification (default).
+ Raises exception if `target_crs` is not LAEA Europe (EPSG:3035).
+ fix_geom : bool
+ If True, invalid geometries are fixed by buffering by the value
+ specified in the global config (geodata -> fix_geom_buffer).
+ add_id_column : bool
+ If True, data is reindexed starting from 0 and a new column "id" is
+ added with the same values.
+
+ Returns
+ -------
+ gpd.GeoDataFrame
+ Processed geodata
+ """
+
+ def check_crs(operation: str) -> None:
+ """Check if requested CRS is LAEA Europe (EPSG:3035)"""
+ if target_crs.lower() != "epsg:3035":
+ raise ValueError(
+ f"Cannot apply {operation} in non-equistant CRS "
+ f"(requested CRS: {target_crs.lower()}) !"
+ )
+
+ # Transform to target CRS
+ if gdf.crs is not None:
+ if str(gdf.crs).lower() != target_crs.lower():
+ gdf = gdf.to_crs(target_crs)
+ else:
+ raise ValueError("Geodata has not CRS assigned.")
+
+ # Filter by min size
+ if min_size is not None:
+ check_crs("min size filtering")
+ gdf = gdf[gdf.area > min_size]
+
+ # Generalize
+ if simplify_tol is not None:
+ check_crs("simplification")
+ gdf["geometry"] = gdf.simplify(simplify_tol, preserve_topology=True)
+
+ # Fix invalid geometries
+ if fix_geom is True:
+ buffer = GLOBAL_CONFIG["global"]["geodata"]["fix_geom_buffer"]
+ if buffer > 0:
+ gdf["geometry"] = gdf.buffer(buffer)
+
+ # Reindex starting from 0 and add new column "id" with same values
+ if add_id_column is True:
+ gdf.reset_index(drop=True, inplace=True)
+ gdf = gdf.assign(id=gdf.index)
+
+ return gdf
+
+
+def overlay(
+ gdf: gpd.GeoDataFrame,
+ gdf_overlay: gpd.GeoDataFrame,
+ retain_rename_overlay_columns: dict = None,
+ gdf_use_centroid: bool = False,
+) -> gpd.GeoDataFrame:
+ """Clips geodata to polygon
+
+ Parameters
+ ----------
+ gdf : gpd.GeoDataFrame
+ Geodata to be clipped (geometry in column "geometry")
+ gdf_overlay : gpd.GeoDataFrame
+ Geodata to clip `gdf` to, e.g. municipalities (geometry in column
+ "geometry")
+ retain_rename_overlay_columns : dict
+ Columns to retain from `gdf_clip` (do not include "geometry")
+ gdf_use_centroid : bool
+ If True, the centroid of gdf will be used for overlay (geometry column
+ will be retained though). Defaults to False.
+ """
+ if retain_rename_overlay_columns is None:
+ columns = ["geometry"]
+ retain_rename_overlay_columns = {}
+ else:
+ if "geometry" in retain_rename_overlay_columns.keys():
+ raise ValueError("Geometry must not be in rename dict!")
+ columns = list(retain_rename_overlay_columns.keys()) + ["geometry"]
+
+ # Use centroid if requested
+ if gdf_use_centroid is True:
+ # Retain geometry
+ geometry_backup = gdf.geometry.copy()
+
+ # Clip and rename columns
+ gdf_clipped = (
+ gpd.overlay(
+ gdf.assign(geometry=gdf.centroid),
+ gdf_overlay[columns],
+ how="intersection",
+ )
+ .rename(columns=retain_rename_overlay_columns)
+ .assign(geometry=geometry_backup)
+ )
+ else:
+ # Clip and rename columns
+ gdf_clipped = gpd.overlay(
+ gdf, gdf_overlay[columns], how="intersection"
+ ).rename(columns=retain_rename_overlay_columns)
+
+ return gdf_clipped
+
+
+def clip_raster(
+ raster_file_in: str, clip_file: str, raster_file_out: str
+) -> None:
+ """Clip raster data using vector data
+
+ Parameters
+ ----------
+ raster_file_in : str
+ Path to raster file to be clipped
+ clip_file : str
+ Path to vector file used for clipping
+ raster_file_out : str
+ Path to clipped raster file
+
+ Returns
+ -------
+ None
+ """
+ clip_data = gpd.read_file(clip_file).geometry
+ with rio.open(raster_file_in) as f:
+ out_image, out_transform = mask(f, clip_data, crop=True)
+ out_meta = f.meta
+
+ out_meta.update(
+ {
+ "driver": "GTiff",
+ "height": out_image.shape[1],
+ "width": out_image.shape[2],
+ "transform": out_transform,
+ }
+ )
+ with rio.open(raster_file_out, "w", **out_meta) as dest:
+ dest.write(out_image)
+
+
+def raster_zonal_stats(
+ raster_file_in: str,
+ clip_file: str,
+ zonal_file_out: str,
+ var_name: str = "mean",
+ stats: str = "mean",
+) -> None:
+ """Create zonal stats
+
+ Parameters
+ ----------
+ raster_file_in : str
+ Path to raster file with data
+ clip_file : str
+ Path to vector file (zones) used for zonal stats
+ raster_file_out : str
+ Path to output raster file
+ var_name : str
+ Name of variable in output file
+ stats : str
+ Stats to be created, e.g. "mean" or "sum mean"
+
+ Returns
+ -------
+ None
+ """
+ schema_in_geom, schema_in_props = read_schema_from_file(clip_file)
+ clip_data = gpd.read_file(clip_file)
+
+ zonal_stats_df = gpd.GeoDataFrame()
+ zonal_stats_df = (
+ zonal_stats_df.from_features(
+ zonal_stats(
+ clip_data, raster_file_in, stats=stats, geojson_out=True
+ ),
+ crs=3035,
+ )
+ .rename(columns={"sum": var_name})
+ .to_crs(GLOBAL_CONFIG["global"]["geodata"]["crs"])
+ )
+
+ field_types = [
+ (field, dtype)
+ for field, dtype in schema_in_props.items()
+ if field in zonal_stats_df.columns
+ ]
+ field_types.extend([(var_name, "float")])
+
+ schema_out = {
+ "geometry": schema_in_geom,
+ "properties": OrderedDict(field_types),
+ }
+
+ write_geofile(
+ gdf=zonal_stats_df,
+ file=zonal_file_out,
+ layer_name="res",
+ schema=schema_out,
+ )
diff --git a/digipipe/store/3_datasets/TEMPLATE/config.yml b/digipipe/store/3_datasets/TEMPLATE/config.yml
deleted file mode 100644
index dfd2d7b3..00000000
--- a/digipipe/store/3_datasets/TEMPLATE/config.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-"""
-Config file for dataset
-"""
diff --git a/digipipe/store/3_datasets/TEMPLATE/create.py b/digipipe/store/3_datasets/TEMPLATE/create.py
deleted file mode 100644
index cd44ffd6..00000000
--- a/digipipe/store/3_datasets/TEMPLATE/create.py
+++ /dev/null
@@ -1,3 +0,0 @@
-"""
-Process layer: Build new layer from preprocessed data
-"""
diff --git a/digipipe/store/DATASETS.md b/digipipe/store/DATASETS.md
new file mode 100644
index 00000000..79fb033b
--- /dev/null
+++ b/digipipe/store/DATASETS.md
@@ -0,0 +1,177 @@
+# Data pipeline
+
+## Data flow
+
+This section describes the data flow of the pipeline.
+
+(RAW) --> (PREPROCESSED) --> (DATASETS) --> (APPDATA)
+
+**Overview:**
+
+| **Step** | **Directory** | **Description** | **Rule(s) for target** | **Cfg section** |
+|:--------:|-----------------------|-------------------------------------------|------------------------|-----------------|
+| 0 | `store/raw/` | Raw data as downloaded | TBD | TBD |
+| 1 | `store/preprocessed/` | Preprocessed data, 1:1 from (0) | TBD | TBD |
+| 2 | `store/datasets/` | Datasets, n:1 from (0) and (1) | TBD | TBD |
+| 3 | `store/appdata/` | Data ready to be used in the app from (2) | TBD | TBD |
+
+In the following each step is shortly described along a common example use
+case.
+
+**Example data flow:**
+
+[](../../docs/img/datasets/pipeline_dataflow_example.png)
+
+**Snakefile**
+
+As all rules will be
+searched for and included in the main [Snakefile](../workflow/Snakefile),
+they must have unique names. It's a good idea to use the dataset name as prefix,
+e.g. `rule osm_forest_`.
+
+### (0) Raw
+
+Contains immutable raw data as downloaded. In directory `TEMPLATE` there are
+two additional files: [dataset.md](../store/raw/TEMPLATE/dataset.md) (see that file
+for further instructions) and [metadata.json)](../store/raw/TEMPLATE/metadata.json).
+
+Note: Assumptions are to be defined in the scenarios, not the raw data.
+See the scenario readme in [SCENARIOS.md](../../digipipe/scenarios/SCENARIOS.md).
+
+!!! note "Example"
+ - Dataset A: ERA5 weather dataset for Germany
+ - Dataset B: MaStR dataset on renewable generators
+ - Dataset C: Shapefiles of region: federal districts and municipalities
+
+### (1) Preprocessed
+
+Data from `(0) Raw` that has undergone some preprocessing such as:
+
+- Archive extracted
+- CRS transformed (see below for CRS conventions)
+- Fields filtered
+- **But NO merging/combining/clipping of multiple (raw) datasets! This should
+ be done in (2)**
+
+Rules, config and info
+
+- Preprocessing rule(s) for the dataset can be defined in the dataset's Snakefile:
+[preprocessed/TEMPLATE/create.smk](preprocessed/TEMPLATE/create.smk).
+- Subsequently, these rules **must be** included in the module file
+[preprocessed/module.smk](preprocessed/module.smk) to take effect (see
+template in the file).
+- Custom, dataset-specific configuration can be put into the dataset's config
+file [preprocessed/TEMPLATE/config.yml](preprocessed/TEMPLATE/config.yml).
+- The title and description of each dataset are to be gathered in the file
+[preprocessed/TEMPLATE/dataset.md](preprocessed/TEMPLATE/dataset.md).
+
+!!! note "Example"
+
+ - Dataset D: Extracted ERA5 weather dataset for Germany (from dataset A)
+ - Dataset E: Wind energy turbines extracted from MaStR dataset, filter for
+ columns power and geometry (from dataset B)
+ - Dataset F: Federal districts converted to Geopackage file, CRS transformed
+ (from dataset C)
+ - Dataset G: Municipalities converted to Geopackage file, CRS transformed
+ (from dataset C)
+
+### (2) Datasets
+
+Datasets, created from arbitrary combinations of datasets from
+`store/preprocessed/` and/or `store/datasets/`.
+
+Rules, config and info
+
+- Creation rule(s) for the dataset can be defined in the dataset's
+ Snakefile: [datasets/TEMPLATE/create.smk](datasets/TEMPLATE/create.smk).
+- Subsequently, these rules **must be** included in the module file
+ [datasets/module.smk](datasets/module.smk) to take effect (see
+ template in the file).
+- Custom, dataset-specific configuration can be put into the dataset's config
+ file [datasets/TEMPLATE/config.yml](datasets/TEMPLATE/config.yml).
+- The title and description of each dataset are to be gathered in the file
+ [datasets/TEMPLATE/dataset.md](datasets/TEMPLATE/dataset.md).
+- Custom, dataset-specific scripts are located in `scripts`.
+
+!!! note "Example"
+ Using datasets from `store/preprocessed/` and `store/datasets/`:
+
+ - Dataset H: Wind energy turbines in the region of interest (from datasets E+F)
+ - Dataset I: Normalized wind energy feedin timeseries for the region (from
+ datasets D+G)
+ - Dataset J: Federal districts (from dataset F)
+ - Dataset K: Municipalities (from dataset G)
+
+### (3) App data
+
+Data ready to be used in the app / as expected by the app.
+
+While the data from (0), (1) and (2) contain datasets from intermediate steps,
+this directory holds app-ready datasets. See
+[appdata/APPDATA](appdata/APPDATA.md) for further information.
+
+### To which processing step do the different stages of my data belong to?
+
+Often, there are multiple options to assign a dataset. If you're still unsure
+where the different stages of your dataset should be located after having
+read the examples above, the following example may help you:
+
+Let's say you want to integrate some subsets of an OpenStreetMap dataset,
+namely extracting (a) forests and (b) residential structures for a specific
+region. You want both be stored in separate files. This involves (e.g.):
+
+1. Convert the pbf file to a more handy type such as Geopackage
+2. Extract data using OSM tags
+3. Clip with region
+
+First, you would create a new raw dataset in `store/raw/`. Then, you could
+either put steps 1 and 2 in the preprocessing, resulting in two datasets in
+`store/preprocessed/`. For each, you could then perform step 3 in
+`store/datasets/`.
+However, this would imply a redundant execution of step 1. While this is
+basically fine in the terms of the pipeline flow, it might be a better idea to
+apply only step 1 and create one dataset in `store/preprocessed/`.
+Using this dataset, you would create the two extracts in `store/datasets/`.
+Finally, the datasets after performing step 3 would be created in `store/datasets/` as well
+resulting in a total of four datasets in `store/datasets/`.
+
+### Temporary files
+
+**TODO: REVISE**
+
+Temporary files are stored in `store/temp/` by default and, depending on your
+configuration, can get quite large. You can change the directory in
+`config.yml` --> `path` --> `temp`.
+
+## Further notes
+
+### No data files in the repository! But keep the `.gitkeep` files
+
+Make sure **not to commit any data files** located in `store/` to the
+repository (except for the descriptive readme and metadata files). Rules have
+been defined in the `.gitignore` file which should make git omit those
+files but you better don't count on it. Instead, save them in the designated
+directory on the [RLI Wolke](https://wolke.rl-institut.de/f/160572).
+
+Each data directory in the provided templates contain an empty `.gitkeep`
+file. When creating a new dataset, please commit this file too to make sure
+the (empty) data directory is retained.
+
+### Coordinate reference system
+
+Please use LAEA Europe (EPSG:3035) as default CRS when writing geodata.
+
+**TODO: REVISE**
+
+- The files in `store/raw/` can have an arbitrary CRS.
+- In the preprocessing (step 1) it is converted to the CRS specified in the global `config.yml` --> `preprocessing` -->
+ `crs`. It is important to use a equal-area CRS to make sure operations such as buffering work properly. By default,
+ it is set to LAEA Europe (EPSG:3035).
+- The final output is written in CRS specified in the global `config.yml` --> `output` --> `crs`. By default, it is set
+ to WGS84 (EPSG:4326) used by the app.
+
+## HowTos
+
+### Add a new dataset
+
+**TBD**
diff --git a/digipipe/store/3_datasets/TEMPLATE/output/.gitkeep b/digipipe/store/__init__.py
similarity index 100%
rename from digipipe/store/3_datasets/TEMPLATE/output/.gitkeep
rename to digipipe/store/__init__.py
diff --git a/digipipe/store/appdata/APPDATA.md b/digipipe/store/appdata/APPDATA.md
new file mode 100644
index 00000000..76a32d3e
--- /dev/null
+++ b/digipipe/store/appdata/APPDATA.md
@@ -0,0 +1,16 @@
+# App data
+
+App-ready datasets for the
+[Digiplan App](https://github.com/rl-institut-private/digiplan/).
+
+## Structure and content
+
+| Directory | File/dir | Content |
+|-------------|----------------------|-----------------------------------------------|
+| esys | | Energy system topology and scenario |
+| | | |
+| datapackage | datapackage_app.json | [Datapackage for app](datapackage/dataset.md) |
+| | | |
+| metadata | | |
+
+The app data is loaded on app startup.
diff --git a/digipipe/store/appdata/datapackage/config.yml b/digipipe/store/appdata/datapackage/config.yml
new file mode 100644
index 00000000..cced3918
--- /dev/null
+++ b/digipipe/store/appdata/datapackage/config.yml
@@ -0,0 +1,1388 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Datapackage definition
+name: "digiplan_app_datapackage"
+description: "Datapackage for Digiplan app"
+resources:
+ # Base data
+ base_data:
+ scalars:
+ population:
+ description: "Bevölkerung"
+ path: "scalars/population.csv"
+ fields:
+ 2010:
+ description: "Bevölkerung 2010"
+ unit: "-"
+ 2015:
+ description: "Bevölkerung 2015"
+ unit: "-"
+ 2020:
+ description: "Bevölkerung 2020"
+ unit: "-"
+ 2021:
+ description: "Bevölkerung 2021"
+ unit: "-"
+ 2022:
+ description: "Bevölkerung 2022"
+ unit: "-"
+ 2025:
+ description: "Bevölkerung 2025"
+ unit: "-"
+ 2030:
+ description: "Bevölkerung 2030"
+ unit: "-"
+ 2035:
+ description: "Bevölkerung 2035"
+ unit: "-"
+ 2040:
+ description: "Bevölkerung 2040"
+ unit: "-"
+ 2045:
+ description: "Bevölkerung 2045"
+ unit: "-"
+ _source_path:
+ dataset: "population_region"
+ file: "population.csv"
+ employment:
+ description: "Beschäftigte und Betriebe"
+ path: "scalars/employment.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ employees_total:
+ description: "Sozialversicherungspflichtige Beschäftigte gesamt"
+ unit: "-"
+ companies_total:
+ description: "Betriebe gesamt"
+ unit: "-"
+ employees_ind:
+ description: "Sozialversicherungspflichtige Beschäftigte in der Industrie"
+ unit: "-"
+ companies_ind:
+ description: "Industriebetriebe"
+ unit: "-"
+ _source_path:
+ dataset: "employment_region"
+ file: "employment.csv"
+ osm_buildings:
+ description: "Gebäudegrundflächen und regionaler Anteil"
+ path: "scalars/osm_buildings_ground_area.json"
+ fields:
+ building_ground_area_country:
+ description: "Summe Gebäudegrundfläche in Deutschland"
+ unit: "m2"
+ building_ground_area_region:
+ description: "Summe Gebäudegrundfläche in der Region"
+ unit: "m2"
+ building_ground_area_share_region:
+ description: "Regionaler Anteil an Gebäudegrundflächen"
+ unit: "-"
+ _source_path:
+ dataset: "osm_buildings"
+ file: "osm_buildings_ground_area.json"
+ geodata:
+ # Administrative areas
+ bkg_vg250_region:
+ description: "Region"
+ path: "geodata/bkg_vg250_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "bkg_vg250_region"
+ file: "bkg_vg250_region.gpkg"
+ bkg_vg250_districts_region:
+ description: "Landkreise"
+ path: "geodata/bkg_vg250_districts_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "bkg_vg250_districts_region"
+ file: "bkg_vg250_districts_region.gpkg"
+ bkg_vg250_muns_region:
+ description: "Gemeinden"
+ path: "geodata/bkg_vg250_muns_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "bkg_vg250_muns_region"
+ file: "bkg_vg250_muns_region.gpkg"
+
+ # From Djagora
+ air_traffic_control_system_region:
+ description: "Drehfunkfeuer"
+ path: "geodata/air_traffic_control_system_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "air_traffic_control_system_region.gpkg"
+ aviation_region:
+ description: "Luftverkehr"
+ path: "geodata/aviation_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "aviation_region.gpkg"
+ biosphere_reserve_region:
+ description: "Biosphärenreservate"
+ path: "geodata/biosphere_reserve_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "biosphere_reserve_region.gpkg"
+ drinking_water_protection_area_region:
+ description: "Wasserschutzgebiete"
+ path: "geodata/drinking_water_protection_area_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "drinking_water_protection_area_region.gpkg"
+ fauna_flora_habitat_region:
+ description: "Fauna-Flora-Habitate"
+ path: "geodata/fauna_flora_habitat_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "fauna_flora_habitat_region.gpkg"
+ floodplain_region:
+ description: "Überschwemmungsgebiete"
+ path: "geodata/floodplain_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "floodplain_region.gpkg"
+ forest_region:
+ description: "Wälder"
+ path: "geodata/forest_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "forest_region.gpkg"
+ grid_region:
+ description: "Stromnetze (>=110 kV)"
+ path: "geodata/grid_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "grid_region.gpkg"
+ industry_region:
+ description: "Industrie- und Gewerbegebiete"
+ path: "geodata/industry_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "industry_region.gpkg"
+ landscape_protection_area_region:
+ description: "Landschaftsschutzgebiete"
+ path: "geodata/landscape_protection_area_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "landscape_protection_area_region.gpkg"
+ less_favoured_areas_agricultural_region:
+ description: "Benachteiligte Gebiete"
+ path: "geodata/less_favoured_areas_agricultural_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "less_favoured_areas_agricultural_region.gpkg"
+ military_region:
+ description: "Militärische Sperrgebiete und Liegenschaften"
+ path: "geodata/military_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "military_region.gpkg"
+ nature_conservation_area_region:
+ description: "Naturschutzgebiete"
+ path: "geodata/nature_conservation_area_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "nature_conservation_area_region.gpkg"
+ railway_region:
+ description: "Bahnverkehr"
+ path: "geodata/railway_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "railway_region.gpkg"
+ road_region:
+ description: "Straßen"
+ path: "geodata/road_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "road_region.gpkg"
+ road_railway-500m_region:
+ description: "Straßen und Bahnverkehr (500 m Puffer)"
+ path: "geodata/road_railway-500m_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "road_railway-500m_region.gpkg"
+ settlement-0m_region:
+ description: "Siedlungen"
+ path: "geodata/settlement-0m_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "settlement-0m_region.gpkg"
+ soil_quality_high_region:
+ description: "Ackerflächen mit hoher Bodenqualität (Soil Quality Rating >= 40)"
+ path: "geodata/soil_quality_high_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "soil_quality_high_region.gpkg"
+ soil_quality_low_region:
+ description: "Ackerflächen mit geringer Bodenqualität (Soil Quality Rating < 40)"
+ path: "geodata/soil_quality_low_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "soil_quality_low_region.gpkg"
+ special_protection_area_region:
+ description: "Vogelschutzgebiete"
+ path: "geodata/special_protection_area_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "special_protection_area_region.gpkg"
+ water_region:
+ description: "Gewässser"
+ path: "geodata/water_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "rli_pv_wfr_region"
+ file: "water_region.gpkg"
+
+
+ # Energy production
+ production:
+ scalars:
+ #units:
+ biomass:
+ description: "Biomasseanlagen: Installierte Leistung und Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_biomass_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ capacity_net:
+ description: "Nennleistung"
+ unit: "MW"
+ count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_biomass_region"
+ file: "bnetza_mastr_biomass_stats_muns.csv"
+ combustion:
+ description: "Verbrennungskraftwerke: Installierte Leistung und Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_combustion_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ capacity_net:
+ description: "Nennleistung"
+ unit: "MW"
+ count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_combustion_region"
+ file: "bnetza_mastr_combustion_stats_muns.csv"
+ gsgk:
+ description: "Geo- oder Solarthermie-, Grubengas- und Klärschlamm-Anlagen: Installierte Leistung und Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_gsgk_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ capacity_net:
+ description: "Nennleistung"
+ unit: "MW"
+ count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_gsgk_region"
+ file: "bnetza_mastr_gsgk_stats_muns.csv"
+ hydro-ror:
+ description: "Wasserkraftanlagen: Installierte Leistung und Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_hydro_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ capacity_net:
+ description: "Nennleistung"
+ unit: "MW"
+ count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_hydro_region"
+ file: "bnetza_mastr_hydro_stats_muns.csv"
+ solar-pv_ground:
+ description: "PV-Freiflächenanlagen: Installierte Leistung und Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_pv_ground_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ capacity_net:
+ description: "Nennleistung"
+ unit: "MW"
+ count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_pv_ground_region"
+ file: "bnetza_mastr_pv_ground_stats_muns.csv"
+ solar-pv_ground_development_over_time:
+ description: "Veränderung (pro Jahr) der installierten Gesamtleistung und der Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_pv_ground_development_over_time.csv"
+ fields:
+ year:
+ description: "Jahr"
+ unit: "-"
+ capacity_net:
+ description: "Nennleistung"
+ unit: "MW"
+ unit_count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_pv_ground_region"
+ file: "bnetza_mastr_pv_ground_development_over_time.csv"
+ solar-pv_rooftop:
+ description: "PV-Aufdachanlagen: Installierte Leistung und Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_pv_roof_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ capacity_net:
+ description: "Nennleistung"
+ unit: "MW"
+ count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_pv_roof_region"
+ file: "bnetza_mastr_pv_roof_stats_muns.csv"
+ solar-pv_rooftop_development_over_time:
+ description: "Veränderung (pro Jahr) der installierten Gesamtleistung und der Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_pv_roof_development_over_time.csv"
+ fields:
+ year:
+ description: "Jahr"
+ unit: "-"
+ capacity_net:
+ description: "Nennleistung"
+ unit: "MW"
+ unit_count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_pv_roof_region"
+ file: "bnetza_mastr_pv_roof_development_over_time.csv"
+ storage:
+ description: "Batteriespeicher gesamt: Installierte Leistung und Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_storage_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ storage_capacity:
+ description: "Speicherkapazität"
+ unit: "MWh"
+ count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_storage_region"
+ file: "bnetza_mastr_storage_stats_muns.csv"
+ storage_large:
+ description: "Batteriespeicher groß (>=100 kWh): Installierte Leistung und Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_storage_large_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ storage_capacity:
+ description: "Speicherkapazität"
+ unit: "MWh"
+ count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_storage_region"
+ file: "bnetza_mastr_storage_large_stats_muns.csv"
+ storage_small:
+ description: "Batteriespeicher klein (<100 kWh): Installierte Leistung und Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_storage_small_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ storage_capacity:
+ description: "Speicherkapazität"
+ unit: "MWh"
+ count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_storage_region"
+ file: "bnetza_mastr_storage_small_stats_muns.csv"
+ storage_pv_roof:
+ description: "Spezifische Kapazität und Leistung von Batteriespeichern, die PV-Aufdachanlagen zugeordnet sind (in Betrieb)"
+ path: "scalars/bnetza_mastr_storage_pv_roof.json"
+ fields:
+ pv_roof_share:
+ all_storages:
+ description: "Anteil von PV-Aufdachanlagen mit Speicher (alle Anlagen und Speicher)"
+ unit: "-"
+ home_storages:
+ description: "Anteil von PV-Aufdachanlagen mit Speicher (kleine Anlagen und Speicher)"
+ unit: ""
+ specific_capacity:
+ all_storages:
+ description: "Mittlere spezifische Kapazität aller Speicher"
+ unit: "kWh/kWp"
+ home_storages:
+ description: "Mittlere spezifische Kapazität von kleinen Speichern"
+ unit: "kWh/kWp"
+ specific_power:
+ all_storages:
+ description: "Mittlere spezifische Leistung aller Speicher"
+ unit: "kW/kWp"
+ home_storages:
+ description: "Mittlere spezifische Leistung von kleinen Speichern"
+ unit: "kW/kWp"
+ _source_path:
+ dataset: "bnetza_mastr_storage_region"
+ file: "bnetza_mastr_storage_pv_roof.json"
+ wind-onshore:
+ description: "Windenergieanlagen: Installierte Leistung und Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_wind_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ capacity_net:
+ description: "Nennleistung"
+ unit: "MW"
+ count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_wind_region"
+ file: "bnetza_mastr_wind_stats_muns.csv"
+ wind-onshore_development_over_time:
+ description: "Veränderung (pro Jahr) der installierten Gesamtleistung und der Anzahl (in Betrieb)"
+ path: "scalars/bnetza_mastr_wind_development_over_time.csv"
+ fields:
+ year:
+ description: "Jahr"
+ unit: "-"
+ capacity_net:
+ description: "Nennleistung"
+ unit: "MW"
+ unit_count:
+ description: "Anzahl"
+ unit: "-"
+ _source_path:
+ dataset: "bnetza_mastr_wind_region"
+ file: "bnetza_mastr_wind_development_over_time.csv"
+ technology_data:
+ description: "Volllaststunden je Technologie und Jahr"
+ path: "scalars/technology_data.json"
+ fields:
+ full_load_hours:
+ wind:
+ description: "Volllaststunden Windenergie"
+ unit: "-"
+ pv_ground:
+ description: "Volllaststunden Freiflächen-PV"
+ unit: "-"
+ pv_roof:
+ description: "Volllaststunden Aufdach-PV"
+ unit: "-"
+ st:
+ description: "Volllaststunden Solarthermie"
+ unit: "-"
+ ror:
+ description: "Volllaststunden Laufwasserkraft"
+ unit: "-"
+ bioenergy:
+ description: "Volllaststunden Bioenergie-Stromerzeugung (ohne biogenen Teil des Abfalls)"
+ unit: "-"
+ power_density:
+ wind:
+ description: "Leistungsdichte Windenergie"
+ unit: "MW/km2"
+ pv_ground:
+ description: "Leistungsdichte PV-Freiflächenanlagen"
+ unit: "MW/km2"
+ pv_roof:
+ description: "Leistungsdichte PV-Aufdachanlagen"
+ unit: "MW/km2"
+ st:
+ description: "Leistungsdichte Solarthermie"
+ unit: "MW/km2"
+ nominal_power_per_unit:
+ wind:
+ description: "Nennleistung je Anlage"
+ unit: "MW/Anlage"
+ batteries:
+ large:
+ nominal_power_per_storage_capacity:
+ description: "Speichernennleistung je Speicherkapazität"
+ unit: "kW/kWh"
+ small:
+ nominal_power_per_storage_capacity:
+ description: "Speichernennleistung je Speicherkapazität"
+ unit: "kW/kWh"
+ storage_capacity_per_pv_power:
+ description: "Speicherkapazität je Nennleistung der PV-Anlage"
+ unit: "kWh/kWp"
+ hot_water_storages:
+ large:
+ nominal_power_per_storage_capacity:
+ description: "Speichernennleistung je Speicherkapazität"
+ unit: "kW/kWh"
+ small:
+ nominal_power_per_storage_capacity:
+ description: "Speichernennleistung je Speicherkapazität"
+ unit: "kW/kWh"
+ _source_path:
+ dataset: "technology_data"
+ file: "technology_data.json"
+
+ sequences:
+ solar-pv_ground_profile:
+ description: "Einspeisezeitreihe PV-Freiflächenanlagen, auf 1 normalisiert"
+ path: "sequences/pv_feedin_timeseries.csv"
+ fields:
+ power:
+ description: "Einspeisung"
+ unit: "-"
+ _source_path:
+ dataset: "renewable_feedin"
+ file: "pv_feedin_timeseries.csv"
+ solar-pv_rooftop_profile:
+ description: "Einspeisezeitreihe PV-Aufdachanlagen, auf 1 normalisiert"
+ path: "sequences/pv_feedin_timeseries.csv"
+ fields:
+ power:
+ description: "Einspeisung"
+ unit: "-"
+ _source_path:
+ dataset: "renewable_feedin"
+ file: "pv_feedin_timeseries.csv"
+ solar-thermalcollector_profile:
+ description: "Einspeisezeitreihe Solarthermie, auf 1 normalisiert"
+ path: "sequences/st_feedin_timeseries.csv"
+ fields:
+ power:
+ description: "Einspeisung"
+ unit: "-"
+ _source_path:
+ dataset: "renewable_feedin"
+ file: "st_feedin_timeseries.csv"
+ hydro-ror_profile:
+ description: "Einspeisezeitreihe Wasserkraft, auf 1 normalisiert"
+ path: "sequences/ror_feedin_timeseries.csv"
+ fields:
+ power:
+ description: "Einspeisung"
+ unit: "-"
+ _source_path:
+ dataset: "renewable_feedin"
+ file: "ror_feedin_timeseries.csv"
+ wind-onshore_profile:
+ description: "Einspeisezeitreihe Windenergie, auf 1 normalisiert"
+ path: "sequences/wind_feedin_timeseries.csv"
+ fields:
+ power:
+ description: "Einspeisung"
+ unit: "-"
+ _source_path:
+ dataset: "renewable_feedin"
+ file: "wind_feedin_timeseries.csv"
+ heatpump-cop_profile:
+ description: "Wärmepumpen COP-Zeitreihe"
+ path: "sequences/heatpump_cop_timeseries.csv"
+ fields:
+ cop:
+ description: "COP"
+ unit: "-"
+ _source_path:
+ dataset: "heatpump_cop"
+ file: "heatpump_cop_timeseries.csv"
+
+ geodata:
+ # Generation units
+ bnetza_mastr_biomass_agg_region:
+ description: "Biomasseanlagen (In Betrieb oder geplant)"
+ path: "geodata/bnetza_mastr_biomass_agg_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "bnetza_mastr_biomass_region"
+ file: "bnetza_mastr_biomass_agg_region.gpkg"
+ bnetza_mastr_combustion_agg_region:
+ description: "Verbrennungskraftwerke (In Betrieb oder geplant)"
+ path: "geodata/bnetza_mastr_combustion_agg_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "bnetza_mastr_combustion_region"
+ file: "bnetza_mastr_combustion_agg_region.gpkg"
+ bnetza_mastr_gsgk_agg_region:
+ description: "Geo- oder Solarthermie-, Grubengas- und Klärschlamm-Anlagen (In Betrieb oder geplant)"
+ path: "geodata/bnetza_mastr_gsgk_agg_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "bnetza_mastr_gsgk_region"
+ file: "bnetza_mastr_gsgk_agg_region.gpkg"
+ bnetza_mastr_hydro_agg_region:
+ description: "Wasserkraftanlagen (In Betrieb oder geplant)"
+ path: "geodata/bnetza_mastr_hydro_agg_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "bnetza_mastr_hydro_region"
+ file: "bnetza_mastr_hydro_agg_region.gpkg"
+ bnetza_mastr_pv_ground_agg_region:
+ description: "PV-Freiflächenanlagen (In Betrieb oder geplant)"
+ path: "geodata/bnetza_mastr_pv_ground_agg_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "bnetza_mastr_pv_ground_region"
+ file: "bnetza_mastr_pv_ground_agg_region.gpkg"
+ bnetza_mastr_pv_roof_agg_region:
+ description: "PV-Aufdachanlagen (In Betrieb oder geplant)"
+ path: "geodata/bnetza_mastr_pv_roof_agg_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "bnetza_mastr_pv_roof_region"
+ file: "bnetza_mastr_pv_roof_agg_region.gpkg"
+ bnetza_mastr_storage_agg_region:
+ description: "Batteriespeicher (In Betrieb oder geplant)"
+ path: "geodata/bnetza_mastr_storage_agg_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "bnetza_mastr_storage_region"
+ file: "bnetza_mastr_storage_agg_region.gpkg"
+ bnetza_mastr_wind_agg_region:
+ description: "Windenergieanlagen (In Betrieb oder geplant)"
+ path: "geodata/bnetza_mastr_wind_agg_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "bnetza_mastr_wind_region"
+ file: "bnetza_mastr_wind_agg_region.gpkg"
+ # Energy demand
+ demand:
+ scalars:
+ # Electricity demand
+ electricity-demand_hh:
+ description: "Strombedarf Haushalte"
+ path: "scalars/demand_hh_power_demand.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_electricity_region"
+ file: "demand_hh_power_demand.csv"
+ electricity-demand_cts:
+ description: "Strombedarf GHD"
+ path: "scalars/demand_cts_power_demand.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_electricity_region"
+ file: "demand_cts_power_demand.csv"
+ electricity-demand_ind:
+ description: "Strombedarf Industrie"
+ path: "scalars/demand_ind_power_demand.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_electricity_region"
+ file: "demand_ind_power_demand.csv"
+
+ # Heat demand
+ heat-demand_hh:
+ description: "Wärmebedarf Haushalte gesamt"
+ path: "scalars/demand_hh_heat_demand.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_hh_heat_demand.csv"
+ heat_central-demand_hh:
+ description: "Wärmebedarf Haushalte Fernwärme"
+ path: "scalars/demand_hh_heat_demand_cen.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_hh_heat_demand_cen.csv"
+ heat_decentral-demand_hh:
+ description: "Wärmebedarf Haushalte dezentral"
+ path: "scalars/demand_hh_heat_demand_dec.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_hh_heat_demand_dec.csv"
+ heat-demand_cts:
+ description: "Wärmebedarf GHD gesamt"
+ path: "scalars/demand_cts_heat_demand.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_cts_heat_demand.csv"
+ heat_central-demand_cts:
+ description: "Wärmebedarf GHD Fernwärme"
+ path: "scalars/demand_cts_heat_demand_cen.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_cts_heat_demand_cen.csv"
+ heat_decentral-demand_cts:
+ description: "Wärmebedarf GHD dezentral"
+ path: "scalars/demand_cts_heat_demand_dec.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_cts_heat_demand_dec.csv"
+ heat-demand_ind:
+ description: "Wärmebedarf Industrie gesamt"
+ path: "scalars/demand_ind_heat_demand.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_ind_heat_demand.csv"
+ heat_central-demand_ind:
+ description: "Wärmebedarf Industrie Fernwärme"
+ path: "scalars/demand_ind_heat_demand_cen.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_ind_heat_demand_cen.csv"
+ heat_decentral-demand_ind:
+ description: "Wärmebedarf Industrie dezentral"
+ path: "scalars/demand_ind_heat_demand_dec.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ 2022:
+ description: "Bedarf 2022"
+ unit: "MWh"
+ 2045:
+ description: "Bedarf 2045"
+ unit: "MWh"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_ind_heat_demand_dec.csv"
+
+ # Heating structure
+ heating_structure_decentral:
+ description: "Beheizungsstruktur dezentral (energetische Anteile)"
+ path: "scalars/demand_heat_structure_esys_dec.csv"
+ fields:
+ year:
+ description: "Jahr"
+ unit: "-"
+ carrier:
+ description: "Energieträger"
+ unit: "-"
+ demand_rel:
+ description: "Relativer Bedarf"
+ unit: "-"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_heat_structure_esys_dec.csv"
+ heating_structure_central:
+ description: "Beheizungsstruktur Fernwärme (energetische Anteile)"
+ path: "scalars/demand_heat_structure_esys_cen.csv"
+ fields:
+ year:
+ description: "Jahr"
+ unit: "-"
+ carrier:
+ description: "Energieträger"
+ unit: "-"
+ demand_rel:
+ description: "Relativer Bedarf"
+ unit: "-"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_heat_structure_esys_cen.csv"
+
+ sequences:
+ # Electricity demand profiles
+ electricity-demand_hh_profile:
+ description: "Strombedarfszeitreihe Haushalte, auf 1 normalisiert"
+ path: "sequences/demand_hh_power_timeseries.csv"
+ fields:
+ demand_norm:
+ description: "Bedarf"
+ unit: "-"
+ _source_path:
+ dataset: "demand_electricity_region"
+ file: "demand_hh_power_timeseries.csv"
+ electricity-demand_cts_profile:
+ description: "Strombedarfszeitreihe GHD, auf 1 normalisiert"
+ path: "sequences/demand_cts_power_timeseries.csv"
+ fields:
+ demand_norm:
+ description: "Bedarf"
+ unit: "-"
+ _source_path:
+ dataset: "demand_electricity_region"
+ file: "demand_cts_power_timeseries.csv"
+ electricity-demand_ind_profile:
+ description: "Strombedarfszeitreihe Industrie, auf 1 normalisiert"
+ path: "sequences/demand_ind_power_timeseries.csv"
+ fields:
+ demand_norm:
+ description: "Bedarf"
+ unit: "-"
+ _source_path:
+ dataset: "demand_electricity_region"
+ file: "demand_ind_power_timeseries.csv"
+
+ # Heat demand profiles
+ heat-demand_hh_profile:
+ description: "Wärmebedarfszeitreihe Haushalte, auf 1 normalisiert"
+ path: "sequences/demand_hh_heat_timeseries.csv"
+ fields:
+ demand_norm:
+ description: "Bedarf"
+ unit: "-"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_hh_heat_timeseries.csv"
+ heat-demand_cts_profile:
+ description: "Wärmebedarfszeitreihe GHD, auf 1 normalisiert"
+ path: "sequences/demand_cts_heat_timeseries.csv"
+ fields:
+ demand_norm:
+ description: "Bedarf"
+ unit: "-"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_cts_heat_timeseries.csv"
+ heat-demand_ind_profile:
+ description: "Wärmebedarfszeitreihe Industrie, auf 1 normalisiert"
+ path: "sequences/demand_ind_heat_timeseries.csv"
+ fields:
+ demand_norm:
+ description: "Bedarf"
+ unit: "-"
+ _source_path:
+ dataset: "demand_heat_region"
+ file: "demand_ind_heat_timeseries.csv"
+
+ # Eligible areas for Wind turbines and ground-mounted PV
+ potential_areas:
+ scalars:
+ wind_area_stats:
+ description: "Flächensumme Potenzialflächen Windenergie je Flächentyp"
+ path: "scalars/potentialarea_wind_area_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ stp_2018_vreg:
+ description: "STP Wind 2018 - Vorrang-/Eignungsgebiete"
+ unit: "km2"
+ stp_2027_vr:
+ description: "STP Wind 2027 - Planabsicht Vorranggebiete"
+ unit: "km2"
+ stp_2027_repowering:
+ description: "STP Wind 2027 - Planabsicht Repoweringgebiete"
+ unit: "km2"
+ stp_2027_search_area_forest_area:
+ description: "STP Wind 2027 - Suchraum Wald"
+ unit: "km2"
+ stp_2027_search_area_open_area:
+ description: "STP Wind 2027 - Suchraum Offenland"
+ unit: "km2"
+ _source_path:
+ dataset: "potentialarea_wind_region"
+ file: "potentialarea_wind_area_stats_muns.csv"
+ pv_ground_area_stats:
+ description: "Flächensumme Potenzialflächen PV-Freiflächenanlagen je Flächentyp"
+ path: "scalars/potentialarea_pv_ground_area_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ agriculture_lfa-off_region:
+ description: "Freiflächen-PV auf Acker- und Grünlandflächen mit geringer Bodengüte"
+ unit: "km2"
+ road_railway_region:
+ description: "Potenzialflächen für Freiflächen-PV entlang von Bundesautobahnen und Schienenwegen (500m-Streifen)"
+ unit: "km2"
+ _source_path:
+ dataset: "potentialarea_pv_ground_region"
+ file: "potentialarea_pv_ground_area_stats_muns.csv"
+ pv_ground_area_shares:
+ description: "Anteil Potenzialflächen an Regionsfläche nach Flächentyp"
+ path: "scalars/potentialarea_pv_ground_area_shares.json"
+ fields:
+ road_railway:
+ description: "Anteil Potenzialflächen entlang von Bundesautobahnen und Schienenwegen an Regionsfläche"
+ unit: "-"
+ agri:
+ description: "Anteil Potenzialflächen auf Acker- und Grünlandflächen mit geringer Bodengüte an Regionsfläche"
+ unit: "-"
+ _source_path:
+ dataset: "potentialarea_pv_ground_region"
+ file: "potentialarea_pv_ground_area_shares.json"
+ pv_ground_area_targets:
+ description: "Regionale, aus Bundeszielen anhand der Potenzialflächen desaggregierte Bundesziele für Freiflächen-PV nach Flächentyp"
+ path: "scalars/potentialarea_pv_ground_regionalized_targets.json"
+ fields:
+ target_area_total:
+ description: "Regionales Flächenziel gesamt"
+ unit: "km2"
+ target_area_road_railway:
+ description: "Regionales Flächenziel entlang von Bundesautobahnen und Schienenwegen"
+ unit: "km2"
+ target_area_agri:
+ description: "Regionales Flächenziel auf Acker- und Grünlandflächen mit geringer Bodengüte"
+ unit: "km2"
+ target_power_total:
+ description: "Regionales Leistungsziel gesamt"
+ unit: "MW"
+ target_power_road_railway:
+ description: "Regionales Leistungsziel entlang von Bundesautobahnen und Schienenwegen"
+ unit: "MW"
+ target_power_agri:
+ description: "Regionales Leistungsziel auf Acker- und Grünlandflächen mit geringer Bodengüte"
+ unit: "MW"
+ _source_path:
+ dataset: "potentialarea_pv_ground_region"
+ file: "potentialarea_pv_ground_regionalized_targets.json"
+ pv_roof_area_stats:
+ description: "Potenziale PV-Aufdachanlagen (alle Gebäude)"
+ path: "scalars/potentialarea_pv_roof_area_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ roof_count:
+ description: "Anzahl der Dächer/Gebäude"
+ unit: "-"
+ building_area_sqm:
+ description: "Gesamtgrundfläche"
+ unit: "m2"
+ historic_preservation_count:
+ description: "Anzahl der Gebäude unter Denkmalschutz"
+ unit: "-"
+ installable_power_south:
+ description: "Installierbare Gesamtleistung (Südausrichtung)"
+ unit: "MW"
+ installable_power_north:
+ description: "Installierbare Gesamtleistung (Nordausrichtung)"
+ unit: "MW"
+ installable_power_east:
+ description: "Installierbare Gesamtleistung (Ostausrichtung)"
+ unit: "MW"
+ installable_power_west:
+ description: "Installierbare Gesamtleistung (Westausrichtung)"
+ unit: "MW"
+ installable_power_flat:
+ description: "Installierbare Gesamtleistung (Flachdach)"
+ unit: "MW"
+ installable_power_total:
+ description: "Installierbare Gesamtleistung (Summe)"
+ unit: "MW"
+ energy_annual_south:
+ description: "Erzielbarer Energieertrag (Südausrichtung)"
+ unit: "MWh"
+ energy_annual_north:
+ description: "Erzielbarer Energieertrag (Nordausrichtung)"
+ unit: "MWh"
+ energy_annual_east:
+ description: "Erzielbarer Energieertrag (Ostausrichtung)"
+ unit: "MWh"
+ energy_annual_west:
+ description: "Erzielbarer Energieertrag (Westausrichtung)"
+ unit: "MWh"
+ energy_annual_flat:
+ description: "Erzielbarer Energieertrag (Flachdach)"
+ unit: "MWh"
+ energy_annual_total:
+ description: "Erzielbarer Energieertrag (Summe)"
+ unit: "MWh"
+ _source_path:
+ dataset: "potentialarea_pv_roof_region"
+ file: "potentialarea_pv_roof_area_stats_muns.csv"
+ pv_roof_area_deployment_stats:
+ description: "Genutzte Potenziale PV-Aufdachanlagen (alle Gebäude)"
+ path: "scalars/potentialarea_pv_roof_deployment_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ relative_deployment:
+ description: "Relative Potenzialnutzung"
+ unit: "-"
+ _source_path:
+ dataset: "potentialarea_pv_roof_region"
+ file: "potentialarea_pv_roof_deployment_stats_muns.csv"
+ pv_roof_wo_historic_area_stats:
+ description: "Potenziale PV-Aufdachanlagen (ohne denkmalgeschützte Gebäude)"
+ path: "scalars/potentialarea_pv_roof_wo_historic_area_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ roof_count:
+ description: "Anzahl der Dächer/Gebäude"
+ unit: "-"
+ building_area_sqm:
+ description: "Gesamtgrundfläche"
+ unit: "m2"
+ historic_preservation_count:
+ description: "Anzahl der Gebäude unter Denkmalschutz"
+ unit: "-"
+ installable_power_south:
+ description: "Installierbare Gesamtleistung (Südausrichtung)"
+ unit: "MW"
+ installable_power_north:
+ description: "Installierbare Gesamtleistung (Nordausrichtung)"
+ unit: "MW"
+ installable_power_east:
+ description: "Installierbare Gesamtleistung (Ostausrichtung)"
+ unit: "MW"
+ installable_power_west:
+ description: "Installierbare Gesamtleistung (Westausrichtung)"
+ unit: "MW"
+ installable_power_flat:
+ description: "Installierbare Gesamtleistung (Flachdach)"
+ unit: "MW"
+ installable_power_total:
+ description: "Installierbare Gesamtleistung (Summe)"
+ unit: "MW"
+ energy_annual_south:
+ description: "Erzielbarer Energieertrag (Südausrichtung)"
+ unit: "MWh"
+ energy_annual_north:
+ description: "Erzielbarer Energieertrag (Nordausrichtung)"
+ unit: "MWh"
+ energy_annual_east:
+ description: "Erzielbarer Energieertrag (Ostausrichtung)"
+ unit: "MWh"
+ energy_annual_west:
+ description: "Erzielbarer Energieertrag (Westausrichtung)"
+ unit: "MWh"
+ energy_annual_flat:
+ description: "Erzielbarer Energieertrag (Flachdach)"
+ unit: "MWh"
+ energy_annual_total:
+ description: "Erzielbarer Energieertrag (Summe)"
+ unit: "MWh"
+ _source_path:
+ dataset: "potentialarea_pv_roof_region"
+ file: "potentialarea_pv_roof_wo_historic_area_stats_muns.csv"
+ pv_roof_wo_historic_area_deployment_stats:
+ description: "Genutzte Potenziale PV-Aufdachanlagen (ohne denkmalgeschützte Gebäude)"
+ path: "scalars/potentialarea_pv_roof_wo_historic_deployment_stats_muns.csv"
+ fields:
+ municipality_id:
+ description: "Gemeinde-ID"
+ unit: "-"
+ relative_deployment:
+ description: "Relative Potenzialnutzung"
+ unit: "-"
+ _source_path:
+ dataset: "potentialarea_pv_roof_region"
+ file: "potentialarea_pv_roof_wo_historic_deployment_stats_muns.csv"
+ pv_roof_area_targets:
+ description: "Regionale, aus Bundeszielen anhand der Gebäudegrundflächen desaggregierte Bundesziele für Aufdach-PV"
+ path: "scalars/potentialarea_pv_roof_regionalized_targets.json"
+ fields:
+ target_power_total:
+ description: "Regionales Leistungsziel gesamt"
+ unit: "MW"
+ _source_path:
+ dataset: "potentialarea_pv_roof_region"
+ file: "potentialarea_pv_roof_regionalized_targets.json"
+
+ geodata:
+ potentialarea_pv_agriculture_lfa-off_region:
+ description: "Potenzialflächen Freiflächen-PV auf Ackerflächen inner- und außerhalb benachteiligter Gebiete"
+ path: "geodata/potentialarea_pv_agriculture_lfa-off_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "potentialarea_pv_ground_region"
+ file: "potentialarea_pv_agriculture_lfa-off_region.gpkg"
+ potentialarea_pv_road_railway_region:
+ description: "Potenzialflächen Freiflächen-PV entlang von Autobahnen und Schienenwegen"
+ path: "geodata/potentialarea_pv_road_railway_region.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "potentialarea_pv_ground_region"
+ file: "potentialarea_pv_road_railway_region.gpkg"
+
+ potentialarea_wind_stp_2018_vreg:
+ description: "Potenzialflächen Windenergie: STP Wind 2018 - Vorrang-/Eignungsgebiete"
+ path: "geodata/potentialarea_wind_stp_2018_vreg.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "potentialarea_wind_region"
+ file: "potentialarea_wind_stp_2018_vreg.gpkg"
+ potentialarea_wind_stp_2027_repowering:
+ description: "Potenzialflächen Windenergie: STP Wind 2027 - Planabsicht Repoweringgebiete"
+ path: "geodata/potentialarea_wind_stp_2027_repowering.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "potentialarea_wind_region"
+ file: "potentialarea_wind_stp_2027_repowering.gpkg"
+ potentialarea_wind_stp_2027_search_area_forest_area:
+ description: "Potenzialflächen Windenergie: STP Wind 2027 - Suchraum Wald"
+ path: "geodata/potentialarea_wind_stp_2027_search_area_forest_area.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "potentialarea_wind_region"
+ file: "potentialarea_wind_stp_2027_search_area_forest_area.gpkg"
+ potentialarea_wind_stp_2027_search_area_open_area:
+ description: "Potenzialflächen Windenergie: STP Wind 2027 - Suchraum Offenland"
+ path: "geodata/potentialarea_wind_stp_2027_search_area_open_area.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "potentialarea_wind_region"
+ file: "potentialarea_wind_stp_2027_search_area_open_area.gpkg"
+ potentialarea_wind_stp_2027_vr:
+ description: "Potenzialflächen Windenergie: STP Wind 2027 - Planabsicht Vorranggebiete"
+ path: "geodata/potentialarea_wind_stp_2027_vr.gpkg"
+ fields: [ ]
+ _source_path:
+ dataset: "potentialarea_wind_region"
+ file: "potentialarea_wind_stp_2027_vr.gpkg"
+
+ # Eligible areas for Wind turbines and ground-mounted PV
+ emissions:
+ scalars:
+ emissions:
+ description: "Emissionen für Sachsen-Anhalt und die Region ABW für 1990 und 2019"
+ path: "scalars/emissions.csv"
+ fields:
+ sector:
+ description: "Sektor nach CRF"
+ unit: "-"
+ cat:
+ description: "Kategorie nach CRF"
+ unit: "-"
+ subcat:
+ description: "Unterkategorie nach CRF"
+ unit: "-"
+ name:
+ description: "Bezeichner"
+ unit: "-"
+ st:
+ description: "Emissionen Sachsen-Anhalt"
+ unit: "kt CO2äq"
+ abw:
+ description: "Emissionen Region ABW"
+ unit: "kt CO2äq"
+ _source_path:
+ dataset: "emissions_region"
+ file: "emissions.csv"
+ emissions_chart_overview:
+ description: "Chartdaten: Emissionen für Sachsen-Anhalt und die Region ABW für 1990 und 2019"
+ path: "scalars/emissions_chart_overview.json"
+ fields:
+ energy_industry:
+ description: "Energiewirtschaft"
+ unit: "kt CO2äq"
+ industry:
+ description: "Industrie"
+ unit: "kt CO2äq"
+ traffic:
+ description: "Verkehr"
+ unit: "kt CO2äq"
+ buildings_firing:
+ description: "Sonst. Energie (insbes. Gebäude)"
+ unit: "kt CO2äq"
+ agricultural:
+ description: "Landwirtschaft"
+ unit: "kt CO2äq"
+ waste_waste_water:
+ description: "Abfall und Abwasser"
+ unit: "kt CO2äq"
+ _source_path:
+ dataset: "emissions_region"
+ file: "emissions_chart_overview.json"
+
+ # Settings
+ settings:
+ panels:
+ energy_settings_panel:
+ description: "Einstellungen für Panel Strom"
+ path: "settings/energy_settings_panel.json"
+ fields:
+ s_w_1:
+ description: "Attribute für Slider s_w_1"
+ unit: "-"
+ _source_path:
+ dataset: "app_settings"
+ file: "energy_settings_panel.json"
+ heat_settings_panel:
+ description: "Einstellungen für Panel Wärme"
+ path: "settings/heat_settings_panel.json"
+ fields:
+ w_d_wp_1:
+ description: "Attribute für Slider w_d_wp_1"
+ unit: "-"
+ _source_path:
+ dataset: "app_settings"
+ file: "heat_settings_panel.json"
+ traffic_settings_panel:
+ description: "Einstellungen für Panel Verkehr"
+ path: "settings/traffic_settings_panel.json"
+ fields:
+ v_iv_1:
+ description: "Attribute für Slider v_iv_1"
+ unit: "-"
+ _source_path:
+ dataset: "app_settings"
+ file: "traffic_settings_panel.json"
+ map:
+ map_panel_layer_list:
+ description: "Layerliste (rechtes Panel)"
+ path: "settings/map_panel_layer_list.json"
+ fields:
+ boundaries:
+ description: "Grenzen"
+ unit: "-"
+ energy:
+ description: "Energie"
+ unit: "-"
+ settlements_infrastructure:
+ description: "Siedlungen und Infrastruktur"
+ unit: "-"
+ nature_landscape:
+ description: "Natur und Landschaft"
+ unit: "-"
+ _source_path:
+ dataset: "app_settings"
+ file: "map_panel_layer_list.json"
+
+ # Captions
+ captions:
+ captions:
+ captions_fields:
+ description: "Captions für Datensatzfelder"
+ path: "captions/captions_fields.json"
+ fields:
+ datasets_caption_map:
+ description: "Mapping File -> Eintrag"
+ unit: "-"
+ captions:
+ description: "Eintrag mit Beschriftungstexten"
+ unit: "-"
+ _source_path:
+ dataset: "app_captions"
+ file: "captions_fields.json"
diff --git a/digipipe/store/appdata/datapackage/create.smk b/digipipe/store/appdata/datapackage/create.smk
new file mode 100644
index 00000000..90c5a447
--- /dev/null
+++ b/digipipe/store/appdata/datapackage/create.smk
@@ -0,0 +1,35 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import json
+import shutil
+
+from digipipe.store.utils import get_abs_dataset_path
+from digipipe.store.appdata.datapackage.scripts.create import collect_files
+
+DATASET_PATH = get_abs_dataset_path("appdata", "datapackage", data_dir=True)
+DATAPACKAGE_FILES = collect_files(config, DATASET_PATH)
+
+rule copy_files:
+ """
+ Copy required files
+ """
+ input: DATAPACKAGE_FILES[0]
+ output: DATAPACKAGE_FILES[1]
+ run:
+ print("Copy required files for datapackage...")
+ for src_file, dst_file in zip(*DATAPACKAGE_FILES):
+ shutil.copy(src_file, dst_file)
+
+rule create_datapackage:
+ """
+ Create datapackage for app
+ """
+ input: DATAPACKAGE_FILES[1]
+ output: DATASET_PATH / "datapackage_app.json"
+ run:
+ print("Creating datapackage...")
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(config, f, indent=4)
diff --git a/digipipe/store/4_appdata/data/.gitkeep b/digipipe/store/appdata/datapackage/data/geodata/.gitkeep
similarity index 100%
rename from digipipe/store/4_appdata/data/.gitkeep
rename to digipipe/store/appdata/datapackage/data/geodata/.gitkeep
diff --git a/digipipe/store/4_appdata/metadata/.gitkeep b/digipipe/store/appdata/datapackage/data/scalars/.gitkeep
similarity index 100%
rename from digipipe/store/4_appdata/metadata/.gitkeep
rename to digipipe/store/appdata/datapackage/data/scalars/.gitkeep
diff --git a/digipipe/store/4_appdata/scenarios/.gitkeep b/digipipe/store/appdata/datapackage/data/sequences/.gitkeep
similarity index 100%
rename from digipipe/store/4_appdata/scenarios/.gitkeep
rename to digipipe/store/appdata/datapackage/data/sequences/.gitkeep
diff --git a/digipipe/store/appdata/datapackage/dataset.md b/digipipe/store/appdata/datapackage/dataset.md
new file mode 100644
index 00000000..01288fe2
--- /dev/null
+++ b/digipipe/store/appdata/datapackage/dataset.md
@@ -0,0 +1,25 @@
+# Datapackage für App
+
+Von der App benötigte Daten in JSON `datapackage_app.json`.
+
+Generelle Struktur:
+
+- ``
+ - `` (`scalars`, `sequences` oder `geodata`)
+ - ``
+ - `description`: Beschreibung
+ - `path`: Pfad zur Zieldatei im Datapackage
+ - `fields`: Felder-/Spaltendefinition
+ - `_source_path`: Pfad zur Datei im Quelldatensatz
+ - `dataset`: Name in `store/datasets/`
+ - `file`: Datei
+
+Kategorien bzw. Inhalt `resources`:
+
+- `base_data`: Basisdaten
+- `production`: Energiebereitstellung
+- `demand`: Energiebedarf
+- `potential_areas`: Potenzialgebiete EE
+- `emissions`: Emissionen
+- `settings`: App-Settings
+- `captions`: App-Captions
diff --git a/digipipe/store/appdata/datapackage/scripts/create.py b/digipipe/store/appdata/datapackage/scripts/create.py
new file mode 100644
index 00000000..e9bc3ef8
--- /dev/null
+++ b/digipipe/store/appdata/datapackage/scripts/create.py
@@ -0,0 +1,51 @@
+from pathlib import Path
+from typing import Tuple
+
+from digipipe.store.utils import get_abs_dataset_path
+
+
+def collect_files(
+ config: dict,
+ dataset_path: Path,
+) -> Tuple[list, list]:
+ """Collect paths of source and target files for app datapackage
+
+ Parameters
+ ----------
+ config : dict
+ Dataset config
+ dataset_path: pathlib.Path
+ Path to datapackage
+
+ Returns
+ -------
+ list
+ Source files
+ list
+ Target files
+ """
+ source_files = []
+ target_files = []
+
+ for cat in config["resources"].keys():
+ print(f"Processing {cat} ...")
+ for subcat in config["resources"][cat].keys():
+ print(f" Processing {subcat} ...")
+ for item, data in config["resources"][cat][subcat].items():
+ print(f" Processing {item} ...")
+ source_file = (
+ get_abs_dataset_path(
+ "datasets",
+ data["_source_path"].get("dataset"),
+ data_dir=True,
+ )
+ / data["_source_path"].get("file")
+ )
+ target_file = dataset_path / data.get("path")
+ if target_file not in target_files:
+ source_files.append(source_file)
+ target_files.append(target_file)
+ else:
+ print(" Target file already collected, skipping...")
+
+ return source_files, target_files
diff --git a/digipipe/store/appdata/esys/.gitkeep b/digipipe/store/appdata/esys/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/appdata/metadata/.gitkeep b/digipipe/store/appdata/metadata/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/appdata/module.smk b/digipipe/store/appdata/module.smk
new file mode 100644
index 00000000..943dcda2
--- /dev/null
+++ b/digipipe/store/appdata/module.smk
@@ -0,0 +1,17 @@
+"""
+Dataset registry for appdata module which is loaded by main snakemake file.
+All datasets in the datasets category must be added to this file.
+
+Template:
+---------
+module :
+ snakefile: "/create.smk"
+ config: config["store"]["appdata"][""]
+use rule * from as appdata__*
+
+"""
+
+module datapackage:
+ snakefile: "datapackage/create.smk"
+ config: config["store"]["appdata"]["datapackage"]
+use rule * from datapackage as appdata_datapackage_*
diff --git a/digipipe/store/datasets/TEMPLATE/config.yml b/digipipe/store/datasets/TEMPLATE/config.yml
new file mode 100644
index 00000000..441e79a0
--- /dev/null
+++ b/digipipe/store/datasets/TEMPLATE/config.yml
@@ -0,0 +1,5 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Further custom configuration goes here
diff --git a/digipipe/store/datasets/TEMPLATE/create.smk b/digipipe/store/datasets/TEMPLATE/create.smk
new file mode 100644
index 00000000..d132bdfc
--- /dev/null
+++ b/digipipe/store/datasets/TEMPLATE/create.smk
@@ -0,0 +1,5 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
diff --git a/digipipe/store/datasets/TEMPLATE/data/.gitkeep b/digipipe/store/datasets/TEMPLATE/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/TEMPLATE/dataset.md b/digipipe/store/datasets/TEMPLATE/dataset.md
new file mode 100644
index 00000000..15bd7af4
--- /dev/null
+++ b/digipipe/store/datasets/TEMPLATE/dataset.md
@@ -0,0 +1,4 @@
+# Name des Datensatzes
+
+Eine kurze Beschreibung des Datensatzes.
+Diese hilft der Dokumentation und bei der Erstellung des Berichts.
diff --git a/digipipe/store/datasets/TEMPLATE/scripts/some_script.py b/digipipe/store/datasets/TEMPLATE/scripts/some_script.py
new file mode 100644
index 00000000..6108a38d
--- /dev/null
+++ b/digipipe/store/datasets/TEMPLATE/scripts/some_script.py
@@ -0,0 +1,3 @@
+"""
+Example script to create this dataset
+"""
diff --git a/digipipe/store/datasets/app_captions/config.yml b/digipipe/store/datasets/app_captions/config.yml
new file mode 100644
index 00000000..17d3af5a
--- /dev/null
+++ b/digipipe/store/datasets/app_captions/config.yml
@@ -0,0 +1,3 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
diff --git a/digipipe/store/datasets/app_captions/create.smk b/digipipe/store/datasets/app_captions/create.smk
new file mode 100644
index 00000000..ed27536e
--- /dev/null
+++ b/digipipe/store/datasets/app_captions/create.smk
@@ -0,0 +1,39 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import json
+
+from digipipe.scripts.data_io import load_json
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("datasets", "app_captions", data_dir=True)
+
+rule create_captions:
+ """
+ Create captions for fields in app
+ """
+ input:
+ [
+ rules.datasets_bnetza_mastr_captions_create.output.outfile,
+ rules.datasets_potentialarea_wind_region_create_captions.output[0],
+ rules.datasets_demand_heat_region_create_captions.output[0],
+ ]
+ output:
+ DATASET_PATH / "captions_fields.json"
+ run:
+ captions = dict(
+ datasets_caption_map={},
+ captions={},
+ )
+
+ for file in input:
+ captions_file = load_json(file)
+ captions["datasets_caption_map"].update(
+ captions_file["datasets_caption_map"]
+ )
+ captions["captions"].update(captions_file["captions"])
+
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(captions, f, indent=4)
diff --git a/digipipe/store/datasets/app_captions/data/.gitkeep b/digipipe/store/datasets/app_captions/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/app_captions/dataset.md b/digipipe/store/datasets/app_captions/dataset.md
new file mode 100644
index 00000000..d49a5fc9
--- /dev/null
+++ b/digipipe/store/datasets/app_captions/dataset.md
@@ -0,0 +1,7 @@
+# Captions
+
+Beschriftungen für WebApp.
+
+Dateien:
+
+- Felder: `captions_fields.json`
diff --git a/digipipe/store/datasets/app_settings/config.yml b/digipipe/store/datasets/app_settings/config.yml
new file mode 100644
index 00000000..b6b5fde7
--- /dev/null
+++ b/digipipe/store/datasets/app_settings/config.yml
@@ -0,0 +1,253 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Map list for right panel
+map_panel_layer_list:
+ boundaries:
+ [
+ "bkg_vg250_districts_region",
+ "bkg_vg250_muns_region"
+ ]
+ energy:
+ [
+ "bnetza_mastr_wind_agg_region",
+ "bnetza_mastr_pv_ground_agg_region",
+ "bnetza_mastr_pv_roof_agg_region",
+ "bnetza_mastr_biomass_agg_region",
+ "bnetza_mastr_hydro_agg_region",
+ "bnetza_mastr_combustion_agg_region",
+ "bnetza_mastr_gsgk_agg_region",
+ "bnetza_mastr_storage_agg_region"
+ ]
+ settlements_infrastructure:
+ [
+ "settlement-0m_region",
+ "industry_region",
+ "road_region",
+ "railway_region",
+ "road_railway-500m_region",
+ "aviation_region",
+ "air_traffic_control_system_region",
+ "military_region",
+ "grid_region"
+ ]
+ nature_landscape:
+ [
+ "nature_conservation_area_region",
+ "drinking_water_protection_area_region",
+ "fauna_flora_habitat_region",
+ "special_protection_area_region",
+ "biosphere_reserve_region",
+ "forest_region",
+ "water_region",
+ "floodplain_region",
+ "landscape_protection_area_region",
+ "soil_quality_high_region",
+ "soil_quality_low_region",
+ "less_favoured_areas_agricultural_region"
+ ]
+
+# Settings for energy panel
+panel_settings_templates:
+ energy_settings_panel:
+ s_w_1:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ s_w_3:
+ start: none
+ s_w_4:
+ start: none
+ s_w_4_1:
+ start: none
+ s_w_4_2:
+ start: none
+ s_w_5:
+ start: none
+ s_w_5_1:
+ max: none
+ min: none
+ start: none
+ step: none
+ s_w_5_2:
+ max: none
+ min: none
+ start: none
+ step: none
+ s_pv_ff_1:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ s_pv_ff_3:
+ max: none
+ min: none
+ start: none
+ step: none
+ s_pv_ff_4:
+ max: none
+ min: none
+ start: none
+ step: none
+ s_pv_d_1:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ s_pv_d_3:
+ max: none
+ min: none
+ start: none
+ status_quo: none
+ step: none
+ s_pv_d_4:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ s_h_1:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ disable: none
+ s_v_1:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ s_v_3:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ s_v_4:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ s_v_5:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ s_s_g_1:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+
+ heat_settings_panel:
+ w_d_wp_1:
+ max: none
+ min: none
+ from-min: none
+ from-max: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ w_d_wp_3:
+ max: none
+ min: none
+ from-min: none
+ from-max: none
+ start: none
+ step: none
+ w_d_wp_4:
+ max: none
+ min: none
+ from-min: none
+ from-max: none
+ start: none
+ step: none
+ w_d_wp_5:
+ max: none
+ min: none
+ from-min: none
+ from-max: none
+ start: none
+ step: none
+ w_z_wp_1:
+ max: none
+ min: none
+ from-min: none
+ from-max: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ w_v_1:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ w_v_3:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ w_v_4:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ w_v_5:
+ max: none
+ min: none
+ start: none
+ step: none
+ status_quo: none
+ future_scenario: none
+ w_d_s_1:
+ max: none
+ min: none
+ start: none
+ step: none
+ w_z_s_1:
+ max: none
+ min: none
+ start: none
+ step: none
+
+ traffic_settings_panel:
+ v_iv_1:
+ max: none
+ min: none
+ start: none
+ step: none
+ # status_quo: none
+ # future_scenario: none
+ v_iv_3:
+ max: none
+ min: none
+ start: none
+ step: none
+ # status_quo: none
+ # future_scenario: none
diff --git a/digipipe/store/datasets/app_settings/create.smk b/digipipe/store/datasets/app_settings/create.smk
new file mode 100644
index 00000000..6d9fdc83
--- /dev/null
+++ b/digipipe/store/datasets/app_settings/create.smk
@@ -0,0 +1,135 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import json
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.scripts.data_io import load_json
+from digipipe.store.utils import get_abs_dataset_path
+from digipipe.store.datasets.app_settings.scripts.panels import (
+ PanelSettings,
+ add_electricity_panel_settings,
+ add_heat_panel_settings,
+ add_traffic_panel_settings
+)
+
+DATASET_PATH = get_abs_dataset_path("datasets", "app_settings", data_dir=True)
+
+
+rule create_map_panel_layer_list:
+ """
+ Create layer list for right map panel
+ """
+ input: DATASET_PATH / "energy_settings_panel.json" # Any file (no input required)
+ output: DATASET_PATH / "map_panel_layer_list.json"
+ run:
+ print("Creating list of layers...")
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(config["map_panel_layer_list"], f, indent=4)
+
+rule create_panel_settings:
+ """
+ Create startup settings for settings panels
+ """
+ input:
+ region=rules.datasets_bkg_vg250_region_create.output,
+ tech_data=rules.datasets_technology_data_copy_files.output,
+ wind_stats=rules.datasets_bnetza_mastr_wind_region_create_power_stats_muns.output,
+ wind_area_stats=get_abs_dataset_path(
+ "datasets", "potentialarea_wind_region") / "data" /
+ "potentialarea_wind_area_stats_muns.csv",
+ pv_ground_stats=rules.datasets_bnetza_mastr_pv_ground_region_create_power_stats_muns.output,
+ pv_ground_area_stats=rules.datasets_potentialarea_pv_ground_region_create_area_stats_muns.output,
+ # pv_ground_area_shares=rules.datasets_potentialarea_pv_ground_region_create_potarea_shares.output,
+ pv_ground_targets=rules.datasets_potentialarea_pv_ground_region_regionalize_state_targets.output,
+ pv_roof_stats=rules.datasets_bnetza_mastr_pv_roof_region_create_power_stats_muns.output,
+ pv_roof_area_stats=rules.datasets_potentialarea_pv_roof_region_create_area_stats_muns.output.wo_historic,
+ pv_roof_area_deploy_stats=rules.datasets_potentialarea_pv_roof_region_create_relative_deployment_stats_muns.output,
+ pv_roof_targets=rules.datasets_potentialarea_pv_roof_region_regionalize_state_targets.output,
+ hydro_stats=rules.datasets_bnetza_mastr_hydro_region_create_power_stats_muns.output,
+ demand_hh_power=rules.datasets_demand_electricity_region_hh_merge_demand_years.output.demand,
+ demand_cts_power=rules.datasets_demand_electricity_region_cts_merge_demand_years.output.demand,
+ demand_ind_power=rules.datasets_demand_electricity_region_ind_merge_demand_years.output.demand,
+ storage_large_stats=rules.datasets_bnetza_mastr_storage_region_create_power_stats_muns.output.large,
+ # storage_small_stats=rules.datasets_bnetza_mastr_storage_region_create_power_stats_muns.output.small,
+ storage_pv_roof=rules.datasets_bnetza_mastr_storage_region_create_storage_pv_roof_stats.output,
+
+ heating_structure_decentral=rules.datasets_demand_heat_region_heating_structure_hh_cts.output.heating_structure_esys_dec,
+ heating_structure_central=rules.datasets_demand_heat_region_heating_structure_hh_cts.output.heating_structure_esys_cen,
+ demand_hh_heat=get_abs_dataset_path("datasets", "demand_heat_region") / "data" / "demand_hh_heat_demand.csv",
+ demand_cts_heat=get_abs_dataset_path("datasets", "demand_heat_region") / "data" / "demand_cts_heat_demand.csv",
+ demand_ind_heat=get_abs_dataset_path("datasets", "demand_heat_region") / "data" / "demand_ind_heat_demand.csv",
+
+ output:
+ panel_settings_electricity=DATASET_PATH / "energy_settings_panel.json",
+ panel_settings_heat=DATASET_PATH / "heat_settings_panel.json",
+ panel_settings_traffic=DATASET_PATH / "traffic_settings_panel.json"
+ run:
+ print("Creating electricity panel settings...")
+ panel_settings_electricity = PanelSettings(
+ name="panel_settings_electricity",
+ **config["panel_settings_templates"]["energy_settings_panel"]
+ )
+ panel_settings_electricity = add_electricity_panel_settings(
+ panel_settings_electricity,
+ region=gpd.read_file(input.region[0]),
+ tech_data=load_json(input.tech_data[0]),
+ wind_stats=pd.read_csv(input.wind_stats[0]),
+ wind_area_stats=pd.read_csv(input.wind_area_stats),
+ pv_ground_stats=pd.read_csv(input.pv_ground_stats[0]),
+ pv_ground_area_stats=pd.read_csv(input.pv_ground_area_stats[0], index_col="municipality_id"),
+ # pv_ground_area_shares=load_json(input.pv_ground_area_shares[0]),
+ pv_ground_targets=load_json(input.pv_ground_targets[0]),
+ pv_roof_stats=pd.read_csv(input.pv_roof_stats[0]),
+ pv_roof_area_stats=pd.read_csv(input.pv_roof_area_stats, index_col="municipality_id"),
+ pv_roof_area_deploy_stats=pd.read_csv(input.pv_roof_area_deploy_stats[0]),
+ pv_roof_targets=load_json(input.pv_roof_targets[0]),
+ hydro_stats=pd.read_csv(input.hydro_stats[0]),
+ demand_hh_power=pd.read_csv(input.demand_hh_power, index_col="municipality_id"),
+ demand_cts_power=pd.read_csv(input.demand_cts_power, index_col="municipality_id"),
+ demand_ind_power=pd.read_csv(input.demand_ind_power, index_col="municipality_id"),
+ storage_large_stats=pd.read_csv(input.storage_large_stats),
+ # storage_small_stats=pd.read_csv(input.storage_small_stats),
+ storage_pv_roof=load_json(input.storage_pv_roof[0]),
+ )
+
+ print("Creating heat panel settings...")
+ panel_settings_heat = PanelSettings(
+ name="panel_settings_heat",
+ **config["panel_settings_templates"]["heat_settings_panel"]
+ )
+ panel_settings_heat = add_heat_panel_settings(
+ panel_settings_heat,
+ heating_structure_decentral=pd.read_csv(input.heating_structure_decentral, index_col="year"),
+ heating_structure_central=pd.read_csv(input.heating_structure_central, index_col="year"),
+ demand_hh_heat=pd.read_csv(input.demand_hh_heat, index_col="municipality_id"),
+ demand_cts_heat=pd.read_csv(input.demand_cts_heat, index_col="municipality_id"),
+ demand_ind_heat=pd.read_csv(input.demand_ind_heat, index_col="municipality_id"),
+ )
+
+ print("Creating traffic panel settings...")
+ panel_settings_traffic = PanelSettings(
+ name="panel_settings_traffic",
+ **config["panel_settings_templates"]["traffic_settings_panel"]
+ )
+ panel_settings_traffic = add_traffic_panel_settings(
+ panel_settings_traffic,
+ )
+
+ # Check and dump
+ for panel, file in zip(
+ [panel_settings_electricity,
+ panel_settings_heat,
+ panel_settings_traffic],
+ [output.panel_settings_electricity,
+ output.panel_settings_heat,
+ output.panel_settings_traffic]
+ ):
+ if panel.is_complete():
+ with open(file, "w", encoding="utf8") as f:
+ json.dump(panel.settings, f, indent=4)
+ else:
+ raise ValueError(f"{panel.name} has missing values!")
diff --git a/digipipe/store/datasets/app_settings/data/.gitkeep b/digipipe/store/datasets/app_settings/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/app_settings/dataset.md b/digipipe/store/datasets/app_settings/dataset.md
new file mode 100644
index 00000000..bea77bdb
--- /dev/null
+++ b/digipipe/store/datasets/app_settings/dataset.md
@@ -0,0 +1,70 @@
+# Settings für App
+
+Einstellungen für die App.
+
+## Layerliste (rechtes Panel)
+
+- Konfiguration: [config.yml](https://github.com/rl-institut/digipipe/blob/main/digipipe/store/datasets/app_settings/config.yml) --> `map_panel_layer_list`
+- Ergebnisfile: `map_panel_layer_list.json`
+- Wird manuell in die App eingepflegt (s.
+ [map_config.py](https://github.com/rl-institut/digiplan/blob/main/digiplan/map/map_config.py))
+
+## Settings panels
+
+Die im linken Panel aufgeführten Einstellelemente (Slider und Schalter) werden
+hier parametriert.
+
+- Konfiguration des Templates:
+ [config.yml](https://github.com/rl-institut/digipipe/blob/main/digipipe/store/datasets/app_settings/config.yml) --> `panel_settings_templates`
+- Parametrierung der Slider und Schalter:
+ [panels.py](https://github.com/rl-institut/digipipe/blob/main/digipipe/store/datasets/app_settings/scripts/panels.py)
+- Ergebnisfiles:
+ - `energy_settings_panel.json`
+ - `heat_settings_panel.json`
+ - `traffic_settings_panel.json`
+- Werden in die App eingelesen
+
+### Parametrierung der Einstellelemente
+
+Für die Slider werden folgende Attribute gesetzt:
+Minimum, Maximum, Schrittweite, Startwert, Status-quo-Wert, Zielwert 2045.
+Diese werden wie folgt bestimmt (vgl. auch (i)-Tooltips an den Elementen):
+
+| **Technologie** | **Element id** | **Maximum** | **Startwert** | **Status-quo-Wert** | **Zielwert 2045** |
+|-------------------------|----------------|-------------------------------------|---------------------|-------------------------------------------------|------------------------------------------------|
+| Windenergie | `s_w_1` | Inst. Leistung in bestehenden VR/EG | Wie Status-quo-Wert | Inst. Leistung 2022 | Aus Flächenziel Sachsen-Anhalt (2,2 % in 2032) |
+| | `s_w_3` | - | Wie Status-quo-Wert | On | - |
+| | `s_w_4` | - | Wie Status-quo-Wert | Off | - |
+| | `s_w_4_1` | - | Wie Status-quo-Wert | On | - |
+| | `s_w_4_2` | - | Wie Status-quo-Wert | Off | - |
+| | `s_w_5` | - | Wie Status-quo-Wert | Off | - |
+| | `s_w_5_1` | 100 % | Wie Status-quo-Wert | Theoret. Wert berechnet aus inst. Leistung 2022 | - |
+| | `s_w_5_2` | 100 % | Wie Status-quo-Wert | Theoret. Wert berechnet aus inst. Leistung 2022 | - |
+| Freiflächen-PV | `s_pv_ff_1` | | Wie Status-quo-Wert | Inst. Leistung 2022 | Aus EEG 2023 und regionalen Potenzialen |
+| | `s_pv_ff_3` | 100 % | Wie Status-quo-Wert | Theoret. Wert berechnet aus inst. Leistung 2022 | - |
+| | `s_pv_ff_4` | 100 % | Wie Status-quo-Wert | Theoret. Wert berechnet aus inst. Leistung 2022 | - |
+| Aufdach-PV | `s_pv_d_1` | | Wie Status-quo-Wert | Inst. Leistung 2022 | Aus EEG 2023 und regionalen Potenzialen |
+| | `s_pv_d_3` | 100 % | Wie Status-quo-Wert | Theoret. Wert berechnet aus inst. Leistung 2022 | - |
+| | `s_pv_d_4` | 100 % | Wie Status-quo-Wert | Aus MaStR | - |
+| Wasserkraft | `s_h_1` | Inst. Leistung 2022 | Wie Status-quo-Wert | Inst. Leistung 2022 | Inst. Leistung 2022 |
+| Stromverbrauch | `s_v_1` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `s_v_3` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `s_v_4` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `s_v_5` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| Batterie-Großspeicher | `s_s_g_1` | 50 % | Wie Status-quo-Wert | Aus inst. Kapazität und Einspeisung 2022 | - |
+| | | | Wie Status-quo-Wert | | |
+| WP dezentral | `w_d_wp_1` | 95 % | 50 % | Inst. Leistung 2022 aus BMWK Langfristszenarien | Wert 2045 aus BMWK Langfristszenarien |
+| | `w_d_wp_3` | 95 % | 50 % | - | - |
+| | `w_d_wp_4` | 95 % | 50 % | - | - |
+| | `w_d_wp_5` | 95 % | 50 % | - | - |
+| WP zentral | `w_z_wp_1` | 95 % | 50 % | Inst. Leistung 2022 aus BMWK Langfristszenarien | Wert 2045 aus BMWK Langfristszenarien |
+| Wärmeverbrauch | `w_v_1` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `w_v_3` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `w_v_4` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `w_v_5` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| Wärmespeicher dezentral | `w_d_s_1` | 200 % | 100 % | - | - |
+| Wärmespeicher zentral | `w_z_s_1` | 200 % | 100 % | - | - |
+
+Die Maxima der Regler im Hauptpanel (`s_w_1`, `s_pv_ff_1` usw.) werden in der
+App dynamisch aus den durch die UserInnen vorgenommenen Detaileinstellungen
+(`s_w_3`, `s_pv_ff_1` usw.) berechnet.
diff --git a/digipipe/store/datasets/app_settings/scripts/panels.py b/digipipe/store/datasets/app_settings/scripts/panels.py
new file mode 100644
index 00000000..1fe9447e
--- /dev/null
+++ b/digipipe/store/datasets/app_settings/scripts/panels.py
@@ -0,0 +1,496 @@
+import math
+
+import geopandas as gpd
+import pandas as pd
+
+
+class PanelSettings:
+ """Value store for settings panel"""
+
+ def __init__(self, name, **settings):
+ """Create attributes from yaml file"""
+ self.name = name
+ self.__dict__.update(settings)
+
+ def __str__(self):
+ return self.name
+
+ @property
+ def settings(self):
+ """
+ Make dictionary from value store while omitting non-data attributes
+ """
+ return {k: v for k, v in self.__dict__.items() if k != "name"}
+
+ def update(self, **kwargs):
+ """Updates control element's attributes"""
+
+ def _keys_available(keys, keys_required):
+ return sorted(keys) == sorted(keys_required)
+
+ for control, values in kwargs.items():
+ if hasattr(self, control):
+ values_store = getattr(self, control)
+ if not isinstance(values, dict):
+ raise ValueError("Attributes are no dict.")
+ if _keys_available(values.keys(), values_store.keys()):
+ values_store.update(values)
+ setattr(self, control, values_store)
+ else:
+ raise ValueError(
+ "Attributes in given value dict do not match the "
+ "value store schema! Check the config.yml!"
+ )
+ else:
+ raise ValueError(
+ f"{control} is no control element! Check the config.yml"
+ )
+
+ def is_complete(self):
+ """Returns True if all values are set (no default value "none" left)"""
+ for control, values_store in self.settings.items():
+ for val in values_store.values():
+ if val == "none":
+ print(f"Control {control} is missing at least one value.")
+ return False
+ return True
+
+
+def add_electricity_panel_settings(
+ panel_settings: PanelSettings,
+ region: gpd.GeoDataFrame,
+ tech_data: dict,
+ wind_stats: pd.DataFrame,
+ wind_area_stats: pd.DataFrame,
+ pv_ground_stats: pd.DataFrame,
+ pv_ground_area_stats: pd.DataFrame,
+ # pv_ground_area_shares: dict,
+ pv_roof_stats: pd.DataFrame,
+ pv_roof_area_stats: pd.DataFrame,
+ pv_roof_area_deploy_stats: pd.DataFrame,
+ pv_ground_targets: dict,
+ pv_roof_targets: dict,
+ hydro_stats: pd.DataFrame,
+ demand_hh_power: pd.DataFrame,
+ demand_cts_power: pd.DataFrame,
+ demand_ind_power: pd.DataFrame,
+ storage_large_stats: pd.DataFrame,
+ # storage_small_stats: pd.DataFrame,
+ storage_pv_roof: dict,
+) -> PanelSettings:
+
+ # Wind energy
+ wind_search_area_start = round(
+ wind_stats.capacity_net.sum()
+ / (
+ wind_area_stats[
+ [
+ "stp_2027_search_area_forest_area",
+ "stp_2027_search_area_open_area",
+ ]
+ ]
+ .sum()
+ .sum()
+ * tech_data["power_density"]["wind"]
+ )
+ * 100
+ )
+
+ panel_settings.update(
+ **dict(
+ s_w_1=dict(
+ max=round(
+ wind_area_stats.stp_2018_vreg.sum()
+ * tech_data["power_density"]["wind"]
+ ),
+ min=0,
+ start=round(wind_stats.capacity_net.sum()),
+ step=10,
+ status_quo=round(wind_stats.capacity_net.sum()),
+ future_scenario=round(
+ float(region.area_km2)
+ * 0.022 # Todo: Move to regulation datsset
+ * tech_data["power_density"]["wind"]
+ ),
+ ),
+ s_w_3=dict(start=True),
+ s_w_4=dict(start=False),
+ s_w_4_1=dict(start=True),
+ s_w_4_2=dict(start=False),
+ s_w_5=dict(start=False),
+ s_w_5_1=dict(
+ max=100,
+ min=0,
+ # Use theoretical values as start
+ # to meet the SQ capacity for sake of UX
+ start=wind_search_area_start,
+ step=5,
+ ),
+ s_w_5_2=dict(
+ max=100,
+ min=0,
+ # Use theoretical values as start
+ # to meet the SQ capacity for sake of UX
+ start=wind_search_area_start,
+ step=5,
+ ),
+ )
+ )
+
+ # PV ground and roof
+ pv_ground_search_area_start = math.ceil(
+ pv_ground_stats.capacity_net.sum()
+ / (
+ pv_ground_area_stats.sum().sum()
+ * tech_data["power_density"]["pv_ground"]
+ )
+ * 100
+ )
+ pv_roof_used_start = (
+ pv_roof_area_deploy_stats.capacity_net.sum()
+ / pv_roof_area_deploy_stats.installable_power.sum()
+ * 100
+ )
+ pv_roof_capacity_max = (
+ pv_roof_area_stats[
+ [
+ f"installable_power_{orient}"
+ for orient in ["south", "north", "east", "west", "flat"]
+ ]
+ ]
+ .sum()
+ .sum()
+ )
+ # Check if PV roof potential for current used share setting is smaller
+ # than actual installed capacity (which leads to too small startup value).
+ # If so, correct value by using installed capacity.
+ # The reason for this mismatch are differences in installed and installable
+ # (RPG dataset) capacities.
+ if (pv_roof_capacity_max * math.ceil(pv_roof_used_start) / 100) < round(
+ pv_roof_stats.capacity_net.sum()
+ ):
+ pv_roof_used_start = math.ceil(
+ pv_roof_stats.capacity_net.sum() / pv_roof_capacity_max * 100
+ )
+
+ panel_settings.update(
+ **dict(
+ s_pv_ff_1=dict(
+ max=math.ceil(
+ pv_ground_area_stats.sum().sum()
+ * tech_data["power_density"]["pv_ground"]
+ ),
+ min=0,
+ start=round(pv_ground_stats.capacity_net.sum()),
+ step=10,
+ status_quo=round(pv_ground_stats.capacity_net.sum()),
+ future_scenario=round(pv_ground_targets["target_power_total"]),
+ ),
+ s_pv_ff_3=dict(
+ max=100,
+ min=0,
+ # Use theoretical values as start
+ # to meet the SQ capacity for sake of UX
+ start=pv_ground_search_area_start,
+ step=5,
+ ),
+ s_pv_ff_4=dict(
+ max=100,
+ min=0,
+ # Use theoretical values as start
+ # to meet the SQ capacity for sake of UX
+ start=pv_ground_search_area_start,
+ step=5,
+ ),
+ s_pv_d_1=dict(
+ max=round(pv_roof_capacity_max * pv_roof_used_start / 100),
+ min=0,
+ start=round(pv_roof_stats.capacity_net.sum()),
+ step=10,
+ status_quo=round(pv_roof_stats.capacity_net.sum()),
+ future_scenario=round(pv_roof_targets["target_power_total"]),
+ ),
+ s_pv_d_3=dict(
+ max=50,
+ min=0,
+ start=round(pv_roof_used_start),
+ step=5,
+ status_quo=round(pv_roof_used_start),
+ ),
+ s_pv_d_4=dict(
+ max=100,
+ min=0,
+ start=round(
+ storage_pv_roof["pv_roof_share"]["home_storages"] * 100
+ ),
+ step=5,
+ status_quo=round(
+ storage_pv_roof["pv_roof_share"]["home_storages"] * 100
+ ),
+ ),
+ )
+ )
+
+ # Hydro
+ panel_settings.update(
+ **dict(
+ s_h_1=dict(
+ max=round(hydro_stats.capacity_net.sum()),
+ min=0,
+ start=round(hydro_stats.capacity_net.sum()),
+ step=1,
+ status_quo=round(hydro_stats.capacity_net.sum()),
+ future_scenario=round(hydro_stats.capacity_net.sum()),
+ disable=True,
+ ),
+ )
+ )
+
+ # Demand
+ total_demand = (demand_hh_power + demand_cts_power + demand_ind_power).sum()
+ feedin_wind_pv_daily_mean = (
+ (
+ pv_ground_stats.capacity_net.sum()
+ * tech_data["full_load_hours"]["pv_ground"]["2022"]
+ )
+ + (
+ wind_stats.capacity_net.sum()
+ * tech_data["full_load_hours"]["wind"]["2022"]
+ )
+ ) / 365 # Daily in MWh
+
+ panel_settings.update(
+ **dict(
+ s_v_1=dict(
+ max=200,
+ min=50,
+ start=100,
+ step=10,
+ status_quo=100,
+ future_scenario=round(
+ total_demand["2045"] / total_demand["2022"] * 100
+ ),
+ ),
+ s_v_3=dict(
+ max=200,
+ min=50,
+ start=100,
+ step=10,
+ status_quo=100,
+ future_scenario=round(
+ demand_hh_power.sum()["2045"]
+ / demand_hh_power.sum()["2022"]
+ * 100
+ ),
+ ),
+ s_v_4=dict(
+ max=200,
+ min=50,
+ start=100,
+ step=10,
+ status_quo=100,
+ future_scenario=round(
+ demand_cts_power.sum()["2045"]
+ / demand_cts_power.sum()["2022"]
+ * 100
+ ),
+ ),
+ s_v_5=dict(
+ max=200,
+ min=50,
+ start=100,
+ step=10,
+ status_quo=100,
+ future_scenario=round(
+ demand_ind_power.sum()["2045"]
+ / demand_ind_power.sum()["2022"]
+ * 100
+ ),
+ ),
+ s_s_g_1=dict(
+ max=50,
+ min=0,
+ start=math.ceil(
+ storage_large_stats.storage_capacity.sum()
+ / feedin_wind_pv_daily_mean
+ * 100,
+ ),
+ step=1,
+ status_quo=math.ceil(
+ storage_large_stats.storage_capacity.sum()
+ / feedin_wind_pv_daily_mean
+ * 100,
+ ),
+ ),
+ )
+ )
+
+ return panel_settings
+
+
+def add_heat_panel_settings(
+ panel_settings: PanelSettings,
+ heating_structure_decentral: pd.DataFrame,
+ heating_structure_central: pd.DataFrame,
+ demand_hh_heat: pd.DataFrame,
+ demand_cts_heat: pd.DataFrame,
+ demand_ind_heat: pd.DataFrame,
+) -> PanelSettings:
+
+ # Supply
+ heat_pump_share_dec = heating_structure_decentral.loc[
+ heating_structure_decentral.carrier == "heat_pump"
+ ].demand_rel
+ heat_pump_share_cen = heating_structure_central.loc[
+ heating_structure_central.carrier == "heat_pump"
+ ].demand_rel
+
+ panel_settings.update(
+ **dict(
+ w_d_wp_1={
+ "max": 100,
+ "min": 0,
+ "from-min": 50,
+ "from-max": 95,
+ "start": round(heat_pump_share_dec.loc[2022] * 100),
+ "step": 5,
+ "status_quo": round(heat_pump_share_dec.loc[2022] * 100),
+ "future_scenario": round(heat_pump_share_dec.loc[2045] * 100),
+ },
+ w_d_wp_3={
+ "max": 100,
+ "min": 0,
+ "from-min": 50,
+ "from-max": 95,
+ "start": round(heat_pump_share_dec.loc[2022] * 100),
+ "step": 5,
+ },
+ w_d_wp_4={
+ "max": 100,
+ "min": 0,
+ "from-min": 50,
+ "from-max": 95,
+ "start": round(heat_pump_share_dec.loc[2022] * 100),
+ "step": 5,
+ },
+ w_d_wp_5={
+ "max": 100,
+ "min": 0,
+ "from-min": 50,
+ "from-max": 95,
+ "start": round(heat_pump_share_dec.loc[2022] * 100),
+ "step": 5,
+ },
+ w_z_wp_1={
+ "max": 100,
+ "min": 0,
+ "from-min": 50,
+ "from-max": 95,
+ "start": round(heat_pump_share_cen.loc[2022] * 100),
+ "step": 5,
+ "status_quo": round(heat_pump_share_cen.loc[2022] * 100),
+ "future_scenario": round(heat_pump_share_cen.loc[2045] * 100),
+ },
+ )
+ )
+
+ # Demand
+ total_demand = (demand_hh_heat + demand_cts_heat + demand_ind_heat).sum()
+ panel_settings.update(
+ **dict(
+ w_v_1=dict(
+ max=200,
+ min=50,
+ start=100,
+ step=10,
+ status_quo=100,
+ future_scenario=round(
+ total_demand["2045"] / total_demand["2022"] * 100
+ ),
+ ),
+ w_v_3=dict(
+ max=200,
+ min=50,
+ start=100,
+ step=10,
+ status_quo=100,
+ future_scenario=round(
+ demand_hh_heat.sum()["2045"]
+ / demand_hh_heat.sum()["2022"]
+ * 100
+ ),
+ ),
+ w_v_4=dict(
+ max=200,
+ min=50,
+ start=100,
+ step=10,
+ status_quo=100,
+ future_scenario=round(
+ demand_cts_heat.sum()["2045"]
+ / demand_cts_heat.sum()["2022"]
+ * 100
+ ),
+ ),
+ w_v_5=dict(
+ max=200,
+ min=50,
+ start=100,
+ step=10,
+ status_quo=100,
+ future_scenario=round(
+ demand_ind_heat.sum()["2045"]
+ / demand_ind_heat.sum()["2022"]
+ * 100
+ ),
+ ),
+ )
+ )
+
+ # Storages
+ panel_settings.update(
+ **dict(
+ w_d_s_1=dict(
+ max=200,
+ min=25,
+ start=100,
+ step=5,
+ ),
+ w_z_s_1=dict(
+ max=200,
+ min=25,
+ start=100,
+ step=5,
+ ),
+ )
+ )
+
+ return panel_settings
+
+
+def add_traffic_panel_settings(
+ panel_settings: PanelSettings,
+) -> PanelSettings:
+
+ panel_settings.update(
+ **dict(
+ v_iv_1=dict(
+ max=100,
+ min=0,
+ start=0, # TODO
+ step=5,
+ # status_quo=0, # TODO
+ # future_scenario=0, # TODO
+ ),
+ v_iv_3=dict(
+ max=100,
+ min=0,
+ start=0, # TODO
+ step=5,
+ # status_quo=0, # TODO
+ # future_scenario=0, # TODO
+ ),
+ )
+ )
+
+ return panel_settings
diff --git a/digipipe/store/datasets/bkg_vg250_districts_region/config.yml b/digipipe/store/datasets/bkg_vg250_districts_region/config.yml
new file mode 100644
index 00000000..1cc6ee1b
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_districts_region/config.yml
@@ -0,0 +1,12 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Filter settings
+layer: vg250_krs
+attributes:
+ {"NUTS": "nuts",
+ "GEN": "name",
+ "geometry": "geometry"}
+attributes_filter:
+ {"GF": 4}
diff --git a/digipipe/store/datasets/bkg_vg250_districts_region/create.smk b/digipipe/store/datasets/bkg_vg250_districts_region/create.smk
new file mode 100644
index 00000000..eb5e4372
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_districts_region/create.smk
@@ -0,0 +1,21 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bkg_vg250_districts_region")
+
+rule create:
+ """
+ Extract districts of region
+ """
+ input: rules.preprocessed_bkg_vg250_create.output
+ output: DATASET_PATH / "data" / "bkg_vg250_districts_region.gpkg"
+ params:
+ script=DATASET_PATH / "scripts" / "create.py",
+ config_path=DATASET_PATH / "config.yml"
+ shell:
+ "python {params.script} {input} {params.config_path} {output}"
diff --git a/digipipe/store/datasets/bkg_vg250_districts_region/data/.gitkeep b/digipipe/store/datasets/bkg_vg250_districts_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bkg_vg250_districts_region/dataset.md b/digipipe/store/datasets/bkg_vg250_districts_region/dataset.md
new file mode 100644
index 00000000..48d3e376
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_districts_region/dataset.md
@@ -0,0 +1,4 @@
+# Landkreise
+
+Landkreise der Region aus Geodaten der Verwaltungsgebiete extrahiert und
+nach Landmasse gefiltert (Geofaktor 4 = "mit Struktur Land").
diff --git a/digipipe/store/datasets/bkg_vg250_districts_region/scripts/create.py b/digipipe/store/datasets/bkg_vg250_districts_region/scripts/create.py
new file mode 100644
index 00000000..1d186b74
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_districts_region/scripts/create.py
@@ -0,0 +1,45 @@
+import sys
+
+import geopandas as gpd
+
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.config import read_config
+from digipipe.scripts.geo import (
+ convert_to_multipolygon,
+ rename_filter_attributes,
+ reproject_simplify,
+ write_geofile,
+)
+
+
+def process():
+ data = gpd.read_file(infile, layer=config["layer"])
+ data = rename_filter_attributes(
+ gdf=data,
+ attrs_filter_by_values=config["attributes_filter"],
+ attrs_mapping=config["attributes"],
+ )
+ data = reproject_simplify(
+ gdf=data,
+ add_id_column=True,
+ )
+
+ data = convert_to_multipolygon(data)
+
+ data = data.assign(area_km2=data.area / 1e6)
+
+ write_geofile(
+ gdf=data,
+ file=outfile,
+ layer_name=config["layer"],
+ )
+
+
+if __name__ == "__main__":
+ infile = sys.argv[1]
+ config = read_config(sys.argv[2])
+ config["attributes_filter"]["NUTS"] = GLOBAL_CONFIG["global"]["geodata"][
+ "nuts"
+ ]
+ outfile = sys.argv[3]
+ process()
diff --git a/digipipe/store/datasets/bkg_vg250_federal_states/config.yml b/digipipe/store/datasets/bkg_vg250_federal_states/config.yml
new file mode 100644
index 00000000..ad316c7e
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_federal_states/config.yml
@@ -0,0 +1,12 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Filter settings
+layer: vg250_lan
+attributes:
+ {"NUTS": "nuts",
+ "GEN": "name",
+ "geometry": "geometry"}
+attributes_filter:
+ {"GF": 4}
diff --git a/digipipe/store/datasets/bkg_vg250_federal_states/create.smk b/digipipe/store/datasets/bkg_vg250_federal_states/create.smk
new file mode 100644
index 00000000..e07f135d
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_federal_states/create.smk
@@ -0,0 +1,21 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bkg_vg250_federal_states")
+
+rule create:
+ """
+ Extract land mass of federal states
+ """
+ input: rules.preprocessed_bkg_vg250_create.output
+ output: DATASET_PATH / "data" / "bkg_vg250_federal_states.gpkg"
+ params:
+ script=DATASET_PATH / "scripts" / "create.py",
+ config_path=DATASET_PATH / "config.yml"
+ shell:
+ "python {params.script} {input} {params.config_path} {output}"
diff --git a/digipipe/store/datasets/bkg_vg250_federal_states/data/.gitkeep b/digipipe/store/datasets/bkg_vg250_federal_states/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bkg_vg250_federal_states/dataset.md b/digipipe/store/datasets/bkg_vg250_federal_states/dataset.md
new file mode 100644
index 00000000..9df7cd92
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_federal_states/dataset.md
@@ -0,0 +1,4 @@
+# Bundesländer
+
+Bundesländergrenzen aus Geodaten der Verwaltungsgebiete extrahiert und nach
+Landmasse gefiltert (Geofaktor 4 = "mit Struktur Land").
diff --git a/digipipe/store/datasets/bkg_vg250_federal_states/scripts/create.py b/digipipe/store/datasets/bkg_vg250_federal_states/scripts/create.py
new file mode 100644
index 00000000..35032ca0
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_federal_states/scripts/create.py
@@ -0,0 +1,41 @@
+import sys
+
+import geopandas as gpd
+
+from digipipe.scripts.config import read_config
+from digipipe.scripts.geo import (
+ convert_to_multipolygon,
+ rename_filter_attributes,
+ reproject_simplify,
+ write_geofile,
+)
+
+
+def process():
+ data = gpd.read_file(infile, layer=config["layer"])
+ data = rename_filter_attributes(
+ gdf=data,
+ attrs_filter_by_values=config["attributes_filter"],
+ attrs_mapping=config["attributes"],
+ )
+ data = reproject_simplify(
+ gdf=data,
+ add_id_column=True,
+ )
+
+ data = convert_to_multipolygon(data)
+
+ data = data.assign(area_km2=data.area / 1e6)
+
+ write_geofile(
+ gdf=data,
+ file=outfile,
+ layer_name=config["layer"],
+ )
+
+
+if __name__ == "__main__":
+ infile = sys.argv[1]
+ config = read_config(sys.argv[2])
+ outfile = sys.argv[3]
+ process()
diff --git a/digipipe/store/datasets/bkg_vg250_muns_region/config.yml b/digipipe/store/datasets/bkg_vg250_muns_region/config.yml
new file mode 100644
index 00000000..0f838533
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_muns_region/config.yml
@@ -0,0 +1,12 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Filter settings
+layer: vg250_gem
+attributes:
+ {"AGS": "ags",
+ "GEN": "name",
+ "geometry": "geometry"}
+attributes_filter:
+ {"GF": 4}
diff --git a/digipipe/store/datasets/bkg_vg250_muns_region/create.smk b/digipipe/store/datasets/bkg_vg250_muns_region/create.smk
new file mode 100644
index 00000000..48e756f9
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_muns_region/create.smk
@@ -0,0 +1,19 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path, PATH_TO_REGION_DISTRICTS_GPKG
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bkg_vg250_muns_region")
+
+rule create:
+ """
+ Extract municipalities of region
+ """
+ input:
+ muns=rules.preprocessed_bkg_vg250_create.output,
+ districts=PATH_TO_REGION_DISTRICTS_GPKG
+ output: DATASET_PATH / "data" / "bkg_vg250_muns_region.gpkg"
+ script: DATASET_PATH / "scripts" / "create.py"
diff --git a/digipipe/store/datasets/bkg_vg250_muns_region/data/.gitkeep b/digipipe/store/datasets/bkg_vg250_muns_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bkg_vg250_muns_region/dataset.md b/digipipe/store/datasets/bkg_vg250_muns_region/dataset.md
new file mode 100644
index 00000000..f0b4e7ad
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_muns_region/dataset.md
@@ -0,0 +1,4 @@
+# Gemeinden
+
+Gemeinden der Region aus Geodaten der Verwaltungsgebiete extrahiert und
+nach Landmasse gefiltert (Geofaktor 4 = "mit Struktur Land").
diff --git a/digipipe/store/datasets/bkg_vg250_muns_region/scripts/create.py b/digipipe/store/datasets/bkg_vg250_muns_region/scripts/create.py
new file mode 100644
index 00000000..6a9ff782
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_muns_region/scripts/create.py
@@ -0,0 +1,48 @@
+import geopandas as gpd
+
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.geo import (
+ convert_to_multipolygon,
+ overlay,
+ rename_filter_attributes,
+ reproject_simplify,
+ write_geofile,
+)
+
+
+def process():
+ muns = gpd.read_file(snakemake.input.muns[0], layer=config["layer"])
+ muns = rename_filter_attributes(
+ gdf=muns,
+ attrs_filter_by_values=config["attributes_filter"],
+ attrs_mapping=config["attributes"],
+ )
+ muns = reproject_simplify(
+ gdf=muns,
+ add_id_column=True,
+ )
+
+ muns = overlay(
+ gdf=muns.rename(columns={"id": "municipality_id"}),
+ gdf_overlay=gpd.read_file(snakemake.input.districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ gdf_use_centroid=True,
+ ).rename(columns={"municipality_id": "id"})
+
+ muns = muns.assign(area_km2=muns.area / 1e6)
+
+ muns = convert_to_multipolygon(muns)
+
+ write_geofile(
+ gdf=muns,
+ file=snakemake.output[0],
+ layer_name=config["layer"],
+ )
+
+
+if __name__ == "__main__":
+ config = snakemake.config
+ config["attributes_filter"]["NUTS"] = GLOBAL_CONFIG["global"]["geodata"][
+ "nuts"
+ ]
+ process()
diff --git a/digipipe/store/datasets/bkg_vg250_region/config.yml b/digipipe/store/datasets/bkg_vg250_region/config.yml
new file mode 100644
index 00000000..17d3af5a
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_region/config.yml
@@ -0,0 +1,3 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
diff --git a/digipipe/store/datasets/bkg_vg250_region/create.smk b/digipipe/store/datasets/bkg_vg250_region/create.smk
new file mode 100644
index 00000000..268a9e63
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_region/create.smk
@@ -0,0 +1,21 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bkg_vg250_region")
+
+rule create:
+ """
+ Create region polygon from districts
+ """
+ input: rules.datasets_bkg_vg250_districts_region_create.output
+ output: DATASET_PATH / "data" / "bkg_vg250_region.gpkg"
+ params:
+ script=DATASET_PATH / "scripts" / "create.py",
+ config_path=DATASET_PATH / "config.yml"
+ shell:
+ "python {params.script} {input} {params.config_path} {output}"
diff --git a/digipipe/store/datasets/bkg_vg250_region/data/.gitkeep b/digipipe/store/datasets/bkg_vg250_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bkg_vg250_region/dataset.md b/digipipe/store/datasets/bkg_vg250_region/dataset.md
new file mode 100644
index 00000000..b6e6928f
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_region/dataset.md
@@ -0,0 +1,3 @@
+# Region
+
+Region aus Geodaten der Landkreise zusammengeführt.
diff --git a/digipipe/store/datasets/bkg_vg250_region/scripts/create.py b/digipipe/store/datasets/bkg_vg250_region/scripts/create.py
new file mode 100644
index 00000000..1f12968c
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_region/scripts/create.py
@@ -0,0 +1,33 @@
+import os
+import sys
+
+import geopandas as gpd
+
+from digipipe.scripts.geo import (
+ convert_to_multipolygon,
+ reproject_simplify,
+ write_geofile,
+)
+
+
+def process():
+ data = gpd.read_file(infile)
+ data = gpd.GeoDataFrame(
+ crs=data.crs.srs, geometry=[data.buffer(0.1).unary_union]
+ )
+ data = reproject_simplify(gdf=data, add_id_column=True)
+ data = convert_to_multipolygon(data)
+
+ data = data.assign(area_km2=data.area / 1e6)
+
+ write_geofile(
+ gdf=data,
+ file=outfile,
+ layer_name=os.path.basename(outfile).split(".")[0],
+ )
+
+
+if __name__ == "__main__":
+ infile = sys.argv[1]
+ outfile = sys.argv[3]
+ process()
diff --git a/digipipe/store/datasets/bkg_vg250_state/config.yml b/digipipe/store/datasets/bkg_vg250_state/config.yml
new file mode 100644
index 00000000..e5fb2ad5
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_state/config.yml
@@ -0,0 +1,11 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Filter settings
+layer: vg250_sta
+attributes:
+ {"GEN": "name",
+ "geometry": "geometry"}
+attributes_filter:
+ {"GF": 4}
diff --git a/digipipe/store/datasets/bkg_vg250_state/create.smk b/digipipe/store/datasets/bkg_vg250_state/create.smk
new file mode 100644
index 00000000..35b48129
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_state/create.smk
@@ -0,0 +1,21 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bkg_vg250_state")
+
+rule create:
+ """
+ Extract land mass of Germany
+ """
+ input: rules.preprocessed_bkg_vg250_create.output
+ output: DATASET_PATH / "data" / "bkg_vg250_state.gpkg"
+ params:
+ script=DATASET_PATH / "scripts" / "create.py",
+ config_path=DATASET_PATH / "config.yml"
+ shell:
+ "python {params.script} {input} {params.config_path} {output}"
diff --git a/digipipe/store/datasets/bkg_vg250_state/data/.gitkeep b/digipipe/store/datasets/bkg_vg250_state/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bkg_vg250_state/dataset.md b/digipipe/store/datasets/bkg_vg250_state/dataset.md
new file mode 100644
index 00000000..973fb5fa
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_state/dataset.md
@@ -0,0 +1,4 @@
+# Staat
+
+Staatsgrenze aus Geodaten der Verwaltungsgebiete extrahiert und nach Landmasse
+gefiltert (Geofaktor 4 = "mit Struktur Land").
diff --git a/digipipe/store/datasets/bkg_vg250_state/scripts/create.py b/digipipe/store/datasets/bkg_vg250_state/scripts/create.py
new file mode 100644
index 00000000..35032ca0
--- /dev/null
+++ b/digipipe/store/datasets/bkg_vg250_state/scripts/create.py
@@ -0,0 +1,41 @@
+import sys
+
+import geopandas as gpd
+
+from digipipe.scripts.config import read_config
+from digipipe.scripts.geo import (
+ convert_to_multipolygon,
+ rename_filter_attributes,
+ reproject_simplify,
+ write_geofile,
+)
+
+
+def process():
+ data = gpd.read_file(infile, layer=config["layer"])
+ data = rename_filter_attributes(
+ gdf=data,
+ attrs_filter_by_values=config["attributes_filter"],
+ attrs_mapping=config["attributes"],
+ )
+ data = reproject_simplify(
+ gdf=data,
+ add_id_column=True,
+ )
+
+ data = convert_to_multipolygon(data)
+
+ data = data.assign(area_km2=data.area / 1e6)
+
+ write_geofile(
+ gdf=data,
+ file=outfile,
+ layer_name=config["layer"],
+ )
+
+
+if __name__ == "__main__":
+ infile = sys.argv[1]
+ config = read_config(sys.argv[2])
+ outfile = sys.argv[3]
+ process()
diff --git a/digipipe/store/datasets/bnetza_mastr_biomass_region/config.yml b/digipipe/store/datasets/bnetza_mastr_biomass_region/config.yml
new file mode 100644
index 00000000..789341b8
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_biomass_region/config.yml
@@ -0,0 +1,34 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Filter settings
+attributes:
+ {"EinheitMastrNummer": "mastr_id",
+ "KwkMastrNummer": "kwk_mastr_id",
+ "EinheitBetriebsstatus": "status",
+ "LokationMastrNummer": "mastr_location_id",
+ "Postleitzahl": "zip_code",
+ "Ort": "city",
+ "Laengengrad": "lon",
+ "Breitengrad": "lat",
+ "Inbetriebnahmedatum": "commissioning_date",
+ "GeplantesInbetriebnahmedatum": "commissioning_date_planned",
+ "DatumEndgueltigeStilllegung": "decommissioning_date",
+ "NameStromerzeugungseinheit": "name",
+ "Nettonennleistung": "capacity_net",
+ "Bruttoleistung": "capacity_gross",
+ "ThermischeNutzleistung": "th_capacity",
+ "Einspeisungsart": "feedin_type",
+ "Technologie": "technology",
+ "Hauptbrennstoff": "fuel",
+ "Biomasseart": "fuel_type",
+ "AusschliesslicheVerwendungBiomasse": "biomass_only",
+ "BiogasInanspruchnahmeFlexiPraemie": "flexibility_bonus",
+ "NetzbetreiberpruefungStatus": "validated_by_system_operator"}
+attributes_filter:
+ {"Land": "Deutschland",
+ "EinheitBetriebsstatus": ["In Betrieb", "In Planung"]}
+
+# Output settings
+layer: bnetza_mastr_biomass
diff --git a/digipipe/store/datasets/bnetza_mastr_biomass_region/create.smk b/digipipe/store/datasets/bnetza_mastr_biomass_region/create.smk
new file mode 100644
index 00000000..8418f946
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_biomass_region/create.smk
@@ -0,0 +1,60 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import geopandas as gpd
+
+from digipipe.scripts.datasets.mastr import create_stats_per_municipality
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ PATH_TO_REGION_DISTRICTS_GPKG,
+)
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bnetza_mastr_biomass_region")
+SOURCE_DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "bnetza_mastr", data_dir=True
+)
+
+
+rule create:
+ """
+ Extract biomass units for region
+ """
+ input:
+ units=SOURCE_DATASET_PATH / "bnetza_mastr_biomass_raw.csv",
+ locations=SOURCE_DATASET_PATH
+ / "bnetza_mastr_locations_extended_raw.csv",
+ gridconn=SOURCE_DATASET_PATH / "bnetza_mastr_grid_connections_raw.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG,
+ output:
+ outfile=DATASET_PATH / "data" / "bnetza_mastr_biomass_region.gpkg",
+ outfile_agg=DATASET_PATH
+ / "data"
+ / "bnetza_mastr_biomass_agg_region.gpkg",
+ params:
+ config_file=DATASET_PATH / "config.yml",
+ script:
+ DATASET_PATH / "scripts" / "create.py"
+
+
+rule create_power_stats_muns:
+ """
+ Create stats on installed count of units and power per mun
+ """
+ input:
+ units=DATASET_PATH / "data" / "bnetza_mastr_biomass_region.gpkg",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ output:
+ DATASET_PATH / "data" / "bnetza_mastr_biomass_stats_muns.csv",
+ run:
+ units = create_stats_per_municipality(
+ units_df=gpd.read_file(input.units),
+ muns=gpd.read_file(input.region_muns),
+ column="capacity_net",
+ )
+ units["capacity_net"] = units["capacity_net"].div(1e3) # kW to MW
+ units.to_csv(output[0])
diff --git a/digipipe/store/datasets/bnetza_mastr_biomass_region/data/.gitkeep b/digipipe/store/datasets/bnetza_mastr_biomass_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bnetza_mastr_biomass_region/dataset.md b/digipipe/store/datasets/bnetza_mastr_biomass_region/dataset.md
new file mode 100644
index 00000000..ff6a69c7
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_biomass_region/dataset.md
@@ -0,0 +1,22 @@
+# Biomasse-/Biogasanlagen
+
+Biomasse-/Biogasanlagen in der Region aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_biomass_stats_muns.csv`.
diff --git a/digipipe/store/datasets/bnetza_mastr_biomass_region/scripts/create.py b/digipipe/store/datasets/bnetza_mastr_biomass_region/scripts/create.py
new file mode 100644
index 00000000..ca48097d
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_biomass_region/scripts/create.py
@@ -0,0 +1,117 @@
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.datasets import mastr
+from digipipe.scripts.geo import (
+ overlay,
+ rename_filter_attributes,
+ write_geofile,
+)
+from digipipe.store.utils import (
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ get_names_from_nuts,
+)
+
+
+def process() -> None:
+ attrs = snakemake.config["attributes"]
+ attrs_filter = snakemake.config["attributes_filter"]
+ attrs_filter["Landkreis"] = get_names_from_nuts(
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ GLOBAL_CONFIG["global"]["geodata"]["nuts"],
+ )
+
+ units = pd.read_csv(
+ snakemake.input.units,
+ usecols=set(attrs.keys()) | set(attrs_filter.keys()),
+ dtype={"Postleitzahl": str},
+ )
+
+ units = rename_filter_attributes(
+ gdf=units,
+ attrs_filter_by_values=attrs_filter,
+ attrs_mapping=attrs,
+ ).set_index("mastr_id")
+
+ units = mastr.add_voltage_level(
+ units_df=units,
+ locations_path=snakemake.input.locations,
+ gridconn_path=snakemake.input.gridconn,
+ )
+
+ # Add geometry and drop units without coords and
+ # add column to indicate that location from original data was used
+ units_with_geom = mastr.add_geometry(units)
+ units_with_geom = units_with_geom.assign(
+ geometry_approximated=0,
+ )
+
+ units_without_geom = units.loc[(units.lon.isna() | units.lat.isna())].drop(
+ columns=["lon", "lat"]
+ )
+
+ # Add geometry for all units without coords (<=30 kW) and
+ # add column to indicate that location was inferred by geocoding
+ if len(units_without_geom) > 0:
+ (
+ units_with_inferred_geom_gdf,
+ units_with_inferred_geom_agg_gdf,
+ ) = mastr.geocode_units_wo_geometry(
+ units_without_geom,
+ columns_agg_functions={
+ "capacity_net": ("capacity_net", "sum"),
+ "unit_count": ("capacity_net", "count"),
+ "capacity_gross": ("capacity_gross", "sum"),
+ "th_capacity": ("th_capacity", "sum"),
+ },
+ )
+
+ # Merge both GDFs
+ units = pd.concat([units_with_geom, units_with_inferred_geom_gdf])
+
+ units_agg = pd.concat(
+ [
+ units_with_geom.assign(unit_count=1),
+ units_with_inferred_geom_agg_gdf,
+ ]
+ )
+ else:
+ units = units_with_geom
+ units_agg = units_with_geom.assign(unit_count=1)
+
+ # Clip to region and add mun and district ids
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+
+ write_geofile(
+ gdf=units,
+ file=snakemake.output.outfile,
+ layer_name=snakemake.config["layer"],
+ )
+ write_geofile(
+ gdf=units_agg,
+ file=snakemake.output.outfile_agg,
+ layer_name=snakemake.config["layer"],
+ )
+
+
+process()
diff --git a/digipipe/store/datasets/bnetza_mastr_captions/config.yml b/digipipe/store/datasets/bnetza_mastr_captions/config.yml
new file mode 100644
index 00000000..923c3eee
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_captions/config.yml
@@ -0,0 +1,8 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+additional_captions:
+ {"voltage_level": "Spannungsebene",
+ "geometry_approximated": "Standort genähert",
+ "unit_count": "Anzahl Anlagen"}
diff --git a/digipipe/store/datasets/bnetza_mastr_captions/create.smk b/digipipe/store/datasets/bnetza_mastr_captions/create.smk
new file mode 100644
index 00000000..010af6b9
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_captions/create.smk
@@ -0,0 +1,34 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.scripts.config import load_dataset_configs
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bnetza_mastr_captions")
+
+def get_mastr_configs() -> dict:
+ return {
+ k: v for k, v in load_dataset_configs().get("store")["datasets"].items()
+ if k.startswith("bnetza_mastr")
+ }
+
+
+rule create:
+ """
+ Create name mapping file for MaStR dataset
+ """
+ input:
+ mastr_configs=expand(
+ get_abs_dataset_path("datasets", "{name}",) / "config.yml",
+ name=get_mastr_configs().keys()
+ )
+ output:
+ outfile=DATASET_PATH / "data" / "bnetza_mastr_attribute_captions.json",
+ params:
+ mastr_configs=get_mastr_configs(),
+ additional_captions=config["additional_captions"]
+ script:
+ DATASET_PATH / "scripts" / "create.py"
diff --git a/digipipe/store/datasets/bnetza_mastr_captions/data/.gitkeep b/digipipe/store/datasets/bnetza_mastr_captions/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bnetza_mastr_captions/dataset.md b/digipipe/store/datasets/bnetza_mastr_captions/dataset.md
new file mode 100644
index 00000000..ca7c9313
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_captions/dataset.md
@@ -0,0 +1,5 @@
+# Bezeichner und Namen aus MaStR
+
+Bezeichner und Namen aus MaStR als Mapping ->
+ wobei CamelCase aus in Leerzeichen konvertiert
+werden.
diff --git a/digipipe/store/datasets/bnetza_mastr_captions/scripts/create.py b/digipipe/store/datasets/bnetza_mastr_captions/scripts/create.py
new file mode 100644
index 00000000..f4ada156
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_captions/scripts/create.py
@@ -0,0 +1,34 @@
+import json
+import re
+
+
+def process() -> None:
+ def replace_umlaut(s: str) -> str:
+ replace_dict = {"ae": "ä", "oe": "ö", "ue": "ü", "Mastr": "MaStR"}
+ for k, v in replace_dict.items():
+ s = s.replace(k, v)
+ return s
+
+ # Build mapping dict
+ naming_dict = dict()
+ for mastr_config in snakemake.params.mastr_configs.values():
+ for k, v in mastr_config.get("attributes", {}).items():
+ if naming_dict.get(k, None) is None:
+ naming_dict[v] = replace_umlaut(
+ re.sub("([a-z])([A-Z])", "\g<1> \g<2>", k)
+ )
+ naming_dict.update(snakemake.params.additional_captions)
+
+ naming_dict = {
+ "datasets_caption_map": {
+ _: "mastr" for _ in snakemake.params.mastr_configs.keys()
+ },
+ "captions": {"mastr": naming_dict},
+ }
+
+ # Dump as JSON while preserving umlaute
+ with open(snakemake.output.outfile, "w", encoding="utf8") as f:
+ json.dump(naming_dict, f, ensure_ascii=False, indent=4)
+
+
+process()
diff --git a/digipipe/store/datasets/bnetza_mastr_combustion_region/config.yml b/digipipe/store/datasets/bnetza_mastr_combustion_region/config.yml
new file mode 100644
index 00000000..ce7ed7be
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_combustion_region/config.yml
@@ -0,0 +1,36 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Filter settings
+attributes:
+ {"EinheitMastrNummer": "mastr_id",
+ "KwkMastrNummer": "kwk_mastr_id",
+ "Kraftwerksnummer": "bnetza_id",
+ "EinheitBetriebsstatus": "status",
+ "LokationMastrNummer": "mastr_location_id",
+ "Postleitzahl": "zip_code",
+ "Ort": "city",
+ "Laengengrad": "lon",
+ "Breitengrad": "lat",
+ "Inbetriebnahmedatum": "commissioning_date",
+ "GeplantesInbetriebnahmedatum": "commissioning_date_planned",
+ "DatumEndgueltigeStilllegung": "decommissioning_date",
+ "NameKraftwerk": "name",
+ "NameKraftwerksblock": "block_name",
+ "Einsatzort": "usage_sector",
+ "Nettonennleistung": "capacity_net",
+ "Bruttoleistung": "capacity_gross",
+ "ThermischeNutzleistung": "th_capacity",
+ "Einspeisungsart": "feedin_type",
+ "Technologie": "technology",
+ "Hauptbrennstoff": "fuel_primary",
+ "WeitererHauptbrennstoff": "fuel_secondary",
+ "WeitereBrennstoffe": "fuel_other",
+ "NetzbetreiberpruefungStatus": "validated_by_system_operator"}
+attributes_filter:
+ {"Land": "Deutschland",
+ "EinheitBetriebsstatus": ["In Betrieb", "In Planung"]}
+
+# Output settings
+layer: bnetza_mastr_combustion
diff --git a/digipipe/store/datasets/bnetza_mastr_combustion_region/create.smk b/digipipe/store/datasets/bnetza_mastr_combustion_region/create.smk
new file mode 100644
index 00000000..50bd51c3
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_combustion_region/create.smk
@@ -0,0 +1,62 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import geopandas as gpd
+
+from digipipe.scripts.datasets.mastr import create_stats_per_municipality
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ PATH_TO_REGION_DISTRICTS_GPKG,
+)
+
+DATASET_PATH = get_abs_dataset_path(
+ "datasets", "bnetza_mastr_combustion_region"
+)
+SOURCE_DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "bnetza_mastr", data_dir=True
+)
+
+
+rule create:
+ """
+ Extract combustion units for region
+ """
+ input:
+ units=SOURCE_DATASET_PATH / "bnetza_mastr_combustion_raw.csv",
+ locations=SOURCE_DATASET_PATH
+ / "bnetza_mastr_locations_extended_raw.csv",
+ gridconn=SOURCE_DATASET_PATH / "bnetza_mastr_grid_connections_raw.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG,
+ output:
+ outfile=DATASET_PATH / "data" / "bnetza_mastr_combustion_region.gpkg",
+ outfile_agg=DATASET_PATH
+ / "data"
+ / "bnetza_mastr_combustion_agg_region.gpkg",
+ params:
+ config_file=DATASET_PATH / "config.yml",
+ script:
+ DATASET_PATH / "scripts" / "create.py"
+
+
+rule create_power_stats_muns:
+ """
+ Create stats on installed count of units and power per mun
+ """
+ input:
+ units=DATASET_PATH / "data" / "bnetza_mastr_combustion_region.gpkg",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ output:
+ DATASET_PATH / "data" / "bnetza_mastr_combustion_stats_muns.csv",
+ run:
+ units = create_stats_per_municipality(
+ units_df=gpd.read_file(input.units),
+ muns=gpd.read_file(input.region_muns),
+ column="capacity_net",
+ )
+ units["capacity_net"] = units["capacity_net"].div(1e3) # kW to MW
+ units.to_csv(output[0])
diff --git a/digipipe/store/datasets/bnetza_mastr_combustion_region/data/.gitkeep b/digipipe/store/datasets/bnetza_mastr_combustion_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bnetza_mastr_combustion_region/dataset.md b/digipipe/store/datasets/bnetza_mastr_combustion_region/dataset.md
new file mode 100644
index 00000000..4e7bf5d9
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_combustion_region/dataset.md
@@ -0,0 +1,22 @@
+# Verbrennungskraftwerke
+
+Verbrennungskraftwerke in der Region aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_combustion_stats_muns.csv`.
diff --git a/digipipe/store/datasets/bnetza_mastr_combustion_region/scripts/create.py b/digipipe/store/datasets/bnetza_mastr_combustion_region/scripts/create.py
new file mode 100644
index 00000000..03445983
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_combustion_region/scripts/create.py
@@ -0,0 +1,142 @@
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.datasets import mastr
+from digipipe.scripts.geo import (
+ overlay,
+ rename_filter_attributes,
+ write_geofile,
+)
+from digipipe.store.utils import (
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ df_merge_string_columns,
+ get_names_from_nuts,
+)
+
+
+def process() -> None:
+ attrs = snakemake.config["attributes"]
+ attrs_filter = snakemake.config["attributes_filter"]
+ attrs_filter["Landkreis"] = get_names_from_nuts(
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ GLOBAL_CONFIG["global"]["geodata"]["nuts"],
+ )
+ units = pd.read_csv(
+ snakemake.input.units,
+ usecols=set(attrs.keys()) | set(attrs_filter.keys()),
+ dtype={"Postleitzahl": str},
+ )
+
+ units = rename_filter_attributes(
+ gdf=units,
+ attrs_filter_by_values=attrs_filter,
+ attrs_mapping=attrs,
+ ).set_index("mastr_id")
+
+ units = mastr.add_voltage_level(
+ units_df=units,
+ locations_path=snakemake.input.locations,
+ gridconn_path=snakemake.input.gridconn,
+ )
+
+ # Add geometry and drop units without coords and
+ # add column to indicate that location from original data was used
+ units_with_geom = mastr.add_geometry(units)
+ units_with_geom = units_with_geom.assign(
+ geometry_approximated=0,
+ )
+
+ units_without_geom = units.loc[(units.lon.isna() | units.lat.isna())].drop(
+ columns=["lon", "lat"]
+ )
+
+ # Add geometry for all units without coords (<=30 kW) and
+ # add column to indicate that location was inferred by geocoding
+ if len(units_without_geom) > 0:
+ units_without_geom["fuel_primary"].fillna("", inplace=True)
+ units_without_geom["fuel_secondary"].fillna("", inplace=True)
+
+ (
+ units_with_inferred_geom_gdf,
+ units_with_inferred_geom_agg_gdf,
+ ) = mastr.geocode_units_wo_geometry(
+ units_without_geom,
+ columns_agg_functions={
+ "capacity_net": ("capacity_net", "sum"),
+ "unit_count": ("capacity_net", "count"),
+ "capacity_gross": ("capacity_gross", "sum"),
+ "th_capacity": ("th_capacity", "sum"),
+ "fuel_primary": ("fuel_primary", ";".join),
+ "fuel_secondary": ("fuel_secondary", ";".join),
+ },
+ )
+
+ # Merge fuel types into one and make unique
+ units_with_inferred_geom_agg_gdf["fuels"] = df_merge_string_columns(
+ units_with_inferred_geom_agg_gdf[["fuel_primary", "fuel_secondary"]]
+ )
+
+ # Merge both GDFs
+ units = pd.concat([units_with_geom, units_with_inferred_geom_gdf]).drop(
+ columns=["fuel_primary", "fuel_secondary"]
+ )
+ units_agg = pd.concat(
+ [
+ units_with_geom.assign(
+ unit_count=1,
+ fuels=(
+ units_with_geom["fuel_primary"].fillna("")
+ + "; "
+ + units_with_geom["fuel_secondary"].fillna("")
+ ),
+ ),
+ units_with_inferred_geom_agg_gdf,
+ ]
+ ).drop(columns=["fuel_primary", "fuel_secondary"])
+ else:
+ units = units_with_geom.drop(columns=["fuel_primary", "fuel_secondary"])
+ units_agg = units_with_geom.assign(
+ unit_count=1,
+ fuels=(
+ units_with_geom["fuel_primary"].fillna("")
+ + "; "
+ + units_with_geom["fuel_secondary"].fillna("")
+ ),
+ ).drop(columns=["fuel_primary", "fuel_secondary"])
+
+ # Clip to region and add mun and district ids
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+
+ write_geofile(
+ gdf=units,
+ file=snakemake.output.outfile,
+ layer_name=snakemake.config["layer"],
+ )
+ write_geofile(
+ gdf=units_agg,
+ file=snakemake.output.outfile_agg,
+ layer_name=snakemake.config["layer"],
+ )
+
+
+process()
diff --git a/digipipe/store/datasets/bnetza_mastr_gsgk_region/config.yml b/digipipe/store/datasets/bnetza_mastr_gsgk_region/config.yml
new file mode 100644
index 00000000..51a0fe1d
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_gsgk_region/config.yml
@@ -0,0 +1,31 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Filter settings
+attributes:
+ {"EinheitMastrNummer": "mastr_id",
+ "KwkMastrNummer": "kwk_mastr_id",
+ "EinheitBetriebsstatus": "status",
+ "LokationMastrNummer": "mastr_location_id",
+ "Postleitzahl": "zip_code",
+ "Ort": "city",
+ "Laengengrad": "lon",
+ "Breitengrad": "lat",
+ "Inbetriebnahmedatum": "commissioning_date",
+ "GeplantesInbetriebnahmedatum": "commissioning_date_planned",
+ "DatumEndgueltigeStilllegung": "decommissioning_date",
+ "NameStromerzeugungseinheit": "name",
+ "Nettonennleistung": "capacity_net",
+ "Bruttoleistung": "capacity_gross",
+ "ThermischeNutzleistung": "th_capacity",
+ "Einheittyp": "type",
+ "Einspeisungsart": "feedin_type",
+ "Technologie": "technology",
+ "NetzbetreiberpruefungStatus": "validated_by_system_operator"}
+attributes_filter:
+ {"Land": "Deutschland",
+ "EinheitBetriebsstatus": ["In Betrieb", "In Planung"]}
+
+# Output settings
+layer: bnetza_mastr_gsgk
diff --git a/digipipe/store/datasets/bnetza_mastr_gsgk_region/create.smk b/digipipe/store/datasets/bnetza_mastr_gsgk_region/create.smk
new file mode 100644
index 00000000..e72ac087
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_gsgk_region/create.smk
@@ -0,0 +1,58 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import geopandas as gpd
+
+from digipipe.scripts.datasets.mastr import create_stats_per_municipality
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ PATH_TO_REGION_DISTRICTS_GPKG,
+)
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bnetza_mastr_gsgk_region")
+SOURCE_DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "bnetza_mastr", data_dir=True
+)
+
+
+rule create:
+ """
+ Extract gsgk units for region
+ """
+ input:
+ units=SOURCE_DATASET_PATH / "bnetza_mastr_gsgk_raw.csv",
+ locations=SOURCE_DATASET_PATH
+ / "bnetza_mastr_locations_extended_raw.csv",
+ gridconn=SOURCE_DATASET_PATH / "bnetza_mastr_grid_connections_raw.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG,
+ output:
+ outfile=DATASET_PATH / "data" / "bnetza_mastr_gsgk_region.gpkg",
+ outfile_agg=DATASET_PATH / "data" / "bnetza_mastr_gsgk_agg_region.gpkg",
+ params:
+ config_file=DATASET_PATH / "config.yml",
+ script:
+ DATASET_PATH / "scripts" / "create.py"
+
+
+rule create_power_stats_muns:
+ """
+ Create stats on installed count of units and power per mun
+ """
+ input:
+ units=DATASET_PATH / "data" / "bnetza_mastr_gsgk_region.gpkg",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ output:
+ DATASET_PATH / "data" / "bnetza_mastr_gsgk_stats_muns.csv",
+ run:
+ units = create_stats_per_municipality(
+ units_df=gpd.read_file(input.units),
+ muns=gpd.read_file(input.region_muns),
+ column="capacity_net",
+ )
+ units["capacity_net"] = units["capacity_net"].div(1e3) # kW to MW
+ units.to_csv(output[0])
diff --git a/digipipe/store/datasets/bnetza_mastr_gsgk_region/data/.gitkeep b/digipipe/store/datasets/bnetza_mastr_gsgk_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bnetza_mastr_gsgk_region/dataset.md b/digipipe/store/datasets/bnetza_mastr_gsgk_region/dataset.md
new file mode 100644
index 00000000..27308310
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_gsgk_region/dataset.md
@@ -0,0 +1,23 @@
+# Geo- oder Solarthermie-, Grubengas- und Klärschlamm-Anlagen
+
+Anlagen der Geo- oder Solarthermie, Grubengas und Klärschlamm in der Region
+aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_gsgk_stats_muns.csv`.
diff --git a/digipipe/store/datasets/bnetza_mastr_gsgk_region/scripts/create.py b/digipipe/store/datasets/bnetza_mastr_gsgk_region/scripts/create.py
new file mode 100644
index 00000000..ca48097d
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_gsgk_region/scripts/create.py
@@ -0,0 +1,117 @@
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.datasets import mastr
+from digipipe.scripts.geo import (
+ overlay,
+ rename_filter_attributes,
+ write_geofile,
+)
+from digipipe.store.utils import (
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ get_names_from_nuts,
+)
+
+
+def process() -> None:
+ attrs = snakemake.config["attributes"]
+ attrs_filter = snakemake.config["attributes_filter"]
+ attrs_filter["Landkreis"] = get_names_from_nuts(
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ GLOBAL_CONFIG["global"]["geodata"]["nuts"],
+ )
+
+ units = pd.read_csv(
+ snakemake.input.units,
+ usecols=set(attrs.keys()) | set(attrs_filter.keys()),
+ dtype={"Postleitzahl": str},
+ )
+
+ units = rename_filter_attributes(
+ gdf=units,
+ attrs_filter_by_values=attrs_filter,
+ attrs_mapping=attrs,
+ ).set_index("mastr_id")
+
+ units = mastr.add_voltage_level(
+ units_df=units,
+ locations_path=snakemake.input.locations,
+ gridconn_path=snakemake.input.gridconn,
+ )
+
+ # Add geometry and drop units without coords and
+ # add column to indicate that location from original data was used
+ units_with_geom = mastr.add_geometry(units)
+ units_with_geom = units_with_geom.assign(
+ geometry_approximated=0,
+ )
+
+ units_without_geom = units.loc[(units.lon.isna() | units.lat.isna())].drop(
+ columns=["lon", "lat"]
+ )
+
+ # Add geometry for all units without coords (<=30 kW) and
+ # add column to indicate that location was inferred by geocoding
+ if len(units_without_geom) > 0:
+ (
+ units_with_inferred_geom_gdf,
+ units_with_inferred_geom_agg_gdf,
+ ) = mastr.geocode_units_wo_geometry(
+ units_without_geom,
+ columns_agg_functions={
+ "capacity_net": ("capacity_net", "sum"),
+ "unit_count": ("capacity_net", "count"),
+ "capacity_gross": ("capacity_gross", "sum"),
+ "th_capacity": ("th_capacity", "sum"),
+ },
+ )
+
+ # Merge both GDFs
+ units = pd.concat([units_with_geom, units_with_inferred_geom_gdf])
+
+ units_agg = pd.concat(
+ [
+ units_with_geom.assign(unit_count=1),
+ units_with_inferred_geom_agg_gdf,
+ ]
+ )
+ else:
+ units = units_with_geom
+ units_agg = units_with_geom.assign(unit_count=1)
+
+ # Clip to region and add mun and district ids
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+
+ write_geofile(
+ gdf=units,
+ file=snakemake.output.outfile,
+ layer_name=snakemake.config["layer"],
+ )
+ write_geofile(
+ gdf=units_agg,
+ file=snakemake.output.outfile_agg,
+ layer_name=snakemake.config["layer"],
+ )
+
+
+process()
diff --git a/digipipe/store/datasets/bnetza_mastr_hydro_region/config.yml b/digipipe/store/datasets/bnetza_mastr_hydro_region/config.yml
new file mode 100644
index 00000000..0cfdd71f
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_hydro_region/config.yml
@@ -0,0 +1,30 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Filter settings
+attributes:
+ {"EinheitMastrNummer": "mastr_id",
+ "KwkMastrNummer": "kwk_mastr_id",
+ "EinheitBetriebsstatus": "status",
+ "LokationMastrNummer": "mastr_location_id",
+ "Postleitzahl": "zip_code",
+ "Ort": "city",
+ "Laengengrad": "lon",
+ "Breitengrad": "lat",
+ "Inbetriebnahmedatum": "commissioning_date",
+ "GeplantesInbetriebnahmedatum": "commissioning_date_planned",
+ "DatumEndgueltigeStilllegung": "decommissioning_date",
+ "NameStromerzeugungseinheit": "name",
+ "Nettonennleistung": "capacity_net",
+ "Bruttoleistung": "capacity_gross",
+ "ArtDerWasserkraftanlage": "plant_type",
+ "ArtDesZuflusses": "water_origin",
+ "Einspeisungsart": "feedin_type",
+ "NetzbetreiberpruefungStatus": "validated_by_system_operator"}
+attributes_filter:
+ {"Land": "Deutschland",
+ "EinheitBetriebsstatus": ["In Betrieb", "In Planung"]}
+
+# Output settings
+layer: bnetza_mastr_hydro
diff --git a/digipipe/store/datasets/bnetza_mastr_hydro_region/create.smk b/digipipe/store/datasets/bnetza_mastr_hydro_region/create.smk
new file mode 100644
index 00000000..8429d6a5
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_hydro_region/create.smk
@@ -0,0 +1,58 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import geopandas as gpd
+
+from digipipe.scripts.datasets.mastr import create_stats_per_municipality
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ PATH_TO_REGION_DISTRICTS_GPKG,
+)
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bnetza_mastr_hydro_region")
+SOURCE_DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "bnetza_mastr", data_dir=True
+)
+
+
+rule create:
+ """
+ Extract hydro units for region
+ """
+ input:
+ units=SOURCE_DATASET_PATH / "bnetza_mastr_hydro_raw.csv",
+ locations=SOURCE_DATASET_PATH
+ / "bnetza_mastr_locations_extended_raw.csv",
+ gridconn=SOURCE_DATASET_PATH / "bnetza_mastr_grid_connections_raw.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG,
+ output:
+ outfile=DATASET_PATH / "data" / "bnetza_mastr_hydro_region.gpkg",
+ outfile_agg=DATASET_PATH / "data" / "bnetza_mastr_hydro_agg_region.gpkg",
+ params:
+ config_file=DATASET_PATH / "config.yml",
+ script:
+ DATASET_PATH / "scripts" / "create.py"
+
+
+rule create_power_stats_muns:
+ """
+ Create stats on installed count of units and power per mun
+ """
+ input:
+ units=DATASET_PATH / "data" / "bnetza_mastr_hydro_region.gpkg",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ output:
+ DATASET_PATH / "data" / "bnetza_mastr_hydro_stats_muns.csv",
+ run:
+ units = create_stats_per_municipality(
+ units_df=gpd.read_file(input.units),
+ muns=gpd.read_file(input.region_muns),
+ column="capacity_net",
+ )
+ units["capacity_net"] = units["capacity_net"].div(1e3) # kW to MW
+ units.to_csv(output[0])
diff --git a/digipipe/store/datasets/bnetza_mastr_hydro_region/data/.gitkeep b/digipipe/store/datasets/bnetza_mastr_hydro_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bnetza_mastr_hydro_region/dataset.md b/digipipe/store/datasets/bnetza_mastr_hydro_region/dataset.md
new file mode 100644
index 00000000..bca9d68e
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_hydro_region/dataset.md
@@ -0,0 +1,22 @@
+# Wasserkraftanlagen
+
+Wasserkraftanlagen in der Region aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_hydro_stats_muns.csv`.
diff --git a/digipipe/store/datasets/bnetza_mastr_hydro_region/scripts/create.py b/digipipe/store/datasets/bnetza_mastr_hydro_region/scripts/create.py
new file mode 100644
index 00000000..c2314120
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_hydro_region/scripts/create.py
@@ -0,0 +1,116 @@
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.datasets import mastr
+from digipipe.scripts.geo import (
+ overlay,
+ rename_filter_attributes,
+ write_geofile,
+)
+from digipipe.store.utils import (
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ get_names_from_nuts,
+)
+
+
+def process() -> None:
+ attrs = snakemake.config["attributes"]
+ attrs_filter = snakemake.config["attributes_filter"]
+ attrs_filter["Landkreis"] = get_names_from_nuts(
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ GLOBAL_CONFIG["global"]["geodata"]["nuts"],
+ )
+
+ units = pd.read_csv(
+ snakemake.input.units,
+ usecols=set(attrs.keys()) | set(attrs_filter.keys()),
+ dtype={"Postleitzahl": str},
+ )
+
+ units = rename_filter_attributes(
+ gdf=units,
+ attrs_filter_by_values=attrs_filter,
+ attrs_mapping=attrs,
+ ).set_index("mastr_id")
+
+ units = mastr.add_voltage_level(
+ units_df=units,
+ locations_path=snakemake.input.locations,
+ gridconn_path=snakemake.input.gridconn,
+ )
+
+ # Add geometry and drop units without coords and
+ # add column to indicate that location from original data was used
+ units_with_geom = mastr.add_geometry(units)
+ units_with_geom = units_with_geom.assign(
+ geometry_approximated=0,
+ )
+
+ units_without_geom = units.loc[(units.lon.isna() | units.lat.isna())].drop(
+ columns=["lon", "lat"]
+ )
+
+ # Add geometry for all units without coords (<=30 kW) and
+ # add column to indicate that location was inferred by geocoding
+ if len(units_without_geom) > 0:
+ (
+ units_with_inferred_geom_gdf,
+ units_with_inferred_geom_agg_gdf,
+ ) = mastr.geocode_units_wo_geometry(
+ units_without_geom,
+ columns_agg_functions={
+ "capacity_net": ("capacity_net", "sum"),
+ "unit_count": ("capacity_net", "count"),
+ "capacity_gross": ("capacity_gross", "sum"),
+ },
+ )
+
+ # Merge both GDFs
+ units = pd.concat([units_with_geom, units_with_inferred_geom_gdf])
+
+ units_agg = pd.concat(
+ [
+ units_with_geom.assign(unit_count=1),
+ units_with_inferred_geom_agg_gdf,
+ ]
+ )
+ else:
+ units = units_with_geom
+ units_agg = units_with_geom.assign(unit_count=1)
+
+ # Clip to region and add mun and district ids
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+
+ write_geofile(
+ gdf=units,
+ file=snakemake.output.outfile,
+ layer_name=snakemake.config["layer"],
+ )
+ write_geofile(
+ gdf=units_agg,
+ file=snakemake.output.outfile_agg,
+ layer_name=snakemake.config["layer"],
+ )
+
+
+process()
diff --git a/digipipe/store/datasets/bnetza_mastr_pv_ground_region/config.yml b/digipipe/store/datasets/bnetza_mastr_pv_ground_region/config.yml
new file mode 100644
index 00000000..016a39f3
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_pv_ground_region/config.yml
@@ -0,0 +1,42 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Filter settings
+attributes:
+ {"EinheitMastrNummer": "mastr_id",
+ "EinheitBetriebsstatus": "status",
+ "LokationMastrNummer": "mastr_location_id",
+ "Postleitzahl": "zip_code",
+ "Ort": "city",
+ "Laengengrad": "lon",
+ "Breitengrad": "lat",
+ "Inbetriebnahmedatum": "commissioning_date",
+ "GeplantesInbetriebnahmedatum": "commissioning_date_planned",
+ "DatumEndgueltigeStilllegung": "decommissioning_date",
+ "NameStromerzeugungseinheit": "name",
+ "Bruttoleistung": "capacity_gross",
+ "Nettonennleistung": "capacity_net",
+ "Lage": "site_type",
+ "Einspeisungsart": "feedin_type",
+ "AnzahlModule": "module_count",
+ "Nutzungsbereich": "usage_sector",
+ "Hauptausrichtung": "orientation_primary",
+ "HauptausrichtungNeigungswinkel": "orientation_primary_angle",
+ "Nebenausrichtung": "orientation_secondary",
+ "NebenausrichtungNeigungswinkel": "orientation_secondary_angle",
+ "EinheitlicheAusrichtungUndNeigungswinkel": "orientation_uniform",
+ "Leistungsbegrenzung": "power_limitation",
+ "ArtDerFlaeche": "area_type",
+ "InAnspruchGenommeneFlaeche": "area_occupied",
+ "InAnspruchGenommeneAckerflaeche": "agricultural_area_occupied",
+ "Buergerenergie": "citizens_unit",
+ "MieterstromZugeordnet": "landlord_to_tenant_electricity",
+ "NetzbetreiberpruefungStatus": "validated_by_system_operator"}
+attributes_filter:
+ {"Land": "Deutschland",
+ "EinheitBetriebsstatus": ["In Betrieb", "In Planung"],
+ "Lage": "Freifläche"}
+
+# Output settings
+layer: bnetza_mastr_pv_ground
diff --git a/digipipe/store/datasets/bnetza_mastr_pv_ground_region/create.smk b/digipipe/store/datasets/bnetza_mastr_pv_ground_region/create.smk
new file mode 100644
index 00000000..8322c96d
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_pv_ground_region/create.smk
@@ -0,0 +1,107 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.scripts.datasets.mastr import create_stats_per_municipality
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ PATH_TO_REGION_DISTRICTS_GPKG,
+)
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bnetza_mastr_pv_ground_region")
+SOURCE_DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "bnetza_mastr", data_dir=True
+)
+
+
+rule create:
+ """
+ Extract ground-mounted PV plants for region
+ """
+ input:
+ units=SOURCE_DATASET_PATH / "bnetza_mastr_solar_raw.csv",
+ locations=SOURCE_DATASET_PATH
+ / "bnetza_mastr_locations_extended_raw.csv",
+ gridconn=SOURCE_DATASET_PATH / "bnetza_mastr_grid_connections_raw.csv",
+ unit_correction=get_abs_dataset_path(
+ "raw", "bnetza_mastr_correction_region"
+ )
+ / "data"
+ / "bnetza_mastr_pv_ground_region_correction.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG,
+ output:
+ outfile=DATASET_PATH / "data" / "bnetza_mastr_pv_ground_region.gpkg",
+ outfile_agg=DATASET_PATH
+ / "data"
+ / "bnetza_mastr_pv_ground_agg_region.gpkg",
+ params:
+ config_file=DATASET_PATH / "config.yml",
+ script:
+ DATASET_PATH / "scripts" / "create.py"
+
+
+rule create_power_stats_muns:
+ """
+ Create stats on installed count of units and power per mun
+ """
+ input:
+ units=DATASET_PATH / "data" / "bnetza_mastr_pv_ground_region.gpkg",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ output:
+ DATASET_PATH / "data" / "bnetza_mastr_pv_ground_stats_muns.csv",
+ run:
+ units = create_stats_per_municipality(
+ units_df=gpd.read_file(input.units),
+ muns=gpd.read_file(input.region_muns),
+ column="capacity_net",
+ )
+ units["capacity_net"] = units["capacity_net"].div(1e3) # kW to MW
+ units.to_csv(output[0])
+
+
+rule create_development_over_time:
+ """
+ Create stats on development (per year) of cumulative total installed
+ capacity and cumulative number of operating units
+ """
+ input:
+ agg_region=DATASET_PATH / "data" / "bnetza_mastr_pv_ground_region.gpkg",
+ output:
+ DATASET_PATH
+ / "data"
+ / "bnetza_mastr_pv_ground_development_over_time.csv",
+ run:
+ df = gpd.read_file(input.agg_region)
+ df = df.loc[df["status"] == "In Betrieb"]
+ df["commissioning_date"] = pd.to_datetime(df["commissioning_date"])
+
+ df["year"] = df["commissioning_date"].dt.year
+
+ df_capacity_over_time = (
+ df.groupby("year")["capacity_net"]
+ .sum()
+ .cumsum() # Apply cumulative sum
+ .reset_index()
+ )
+
+ df_units_cumulative = (
+ df.groupby("year")
+ .agg(unit_count=("mastr_id", "count"))
+ .cumsum()
+ .reset_index()
+ )
+ df_combined = df_capacity_over_time.merge(
+ df_units_cumulative, on="year"
+ )
+ df_combined["capacity_net"] = (
+ df_combined["capacity_net"].div(1e3).round(1)
+ )
+ df_combined["year"] = df_combined["year"].astype(int)
+ df_combined.to_csv(output[0], index=False)
diff --git a/digipipe/store/datasets/bnetza_mastr_pv_ground_region/data/.gitkeep b/digipipe/store/datasets/bnetza_mastr_pv_ground_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bnetza_mastr_pv_ground_region/dataset.md b/digipipe/store/datasets/bnetza_mastr_pv_ground_region/dataset.md
new file mode 100644
index 00000000..03025825
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_pv_ground_region/dataset.md
@@ -0,0 +1,30 @@
+# Photovoltaik-Freiflächenanlagen
+
+Photovoltaik-Freiflächenanlagen in der Region aus MaStR-Registerdaten als
+Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_pv_ground_stats_muns.csv`.
+
+## Datenkorrektur
+
+Einige Anlagen sind hinsichtlich Ihrer geografischen Lage oder Typs fehlerhaft.
+Anhand des Datensatzes
+[bnetza_mastr_correction_region](../../raw/bnetza_mastr_correction_region/dataset.md)
+wird für diese Anlagen eine Datenkorrektur vorgenommen.
diff --git a/digipipe/store/datasets/bnetza_mastr_pv_ground_region/scripts/create.py b/digipipe/store/datasets/bnetza_mastr_pv_ground_region/scripts/create.py
new file mode 100644
index 00000000..81b093e3
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_pv_ground_region/scripts/create.py
@@ -0,0 +1,128 @@
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.datasets import mastr
+from digipipe.scripts.geo import (
+ overlay,
+ rename_filter_attributes,
+ write_geofile,
+)
+from digipipe.store.utils import (
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ get_names_from_nuts,
+)
+
+
+def process() -> None:
+ attrs = snakemake.config["attributes"]
+ attrs_filter = snakemake.config["attributes_filter"]
+ attrs_filter["Landkreis"] = get_names_from_nuts(
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ GLOBAL_CONFIG["global"]["geodata"]["nuts"],
+ )
+
+ units = pd.read_csv(
+ snakemake.input.units,
+ usecols=set(attrs.keys()) | set(attrs_filter.keys()),
+ dtype={"Postleitzahl": str},
+ index_col="EinheitMastrNummer",
+ )
+
+ # Apply corrections
+ unit_correction = pd.read_csv(
+ snakemake.input.unit_correction,
+ delimiter=";",
+ index_col="mastr_id",
+ )
+ units = mastr.apply_manual_corrections(
+ units_df=units,
+ units_correction_df=unit_correction,
+ )
+
+ units = rename_filter_attributes(
+ gdf=units,
+ attrs_filter_by_values=attrs_filter,
+ attrs_mapping=attrs,
+ ).set_index("mastr_id")
+
+ units = mastr.add_voltage_level(
+ units_df=units,
+ locations_path=snakemake.input.locations,
+ gridconn_path=snakemake.input.gridconn,
+ )
+
+ # Add geometry and drop units without coords and
+ # add column to indicate that location from original data was used
+ units_with_geom = mastr.add_geometry(units)
+ units_with_geom = units_with_geom.assign(
+ geometry_approximated=0,
+ )
+
+ units_without_geom = units.loc[(units.lon.isna() | units.lat.isna())].drop(
+ columns=["lon", "lat"]
+ )
+
+ # Add geometry for all units without coords (<=30 kW) and
+ # add column to indicate that location was inferred by geocoding
+ if len(units_without_geom) > 0:
+ (
+ units_with_inferred_geom_gdf,
+ units_with_inferred_geom_agg_gdf,
+ ) = mastr.geocode_units_wo_geometry(
+ units_without_geom,
+ columns_agg_functions={
+ "capacity_net": ("capacity_net", "sum"),
+ "unit_count": ("capacity_net", "count"),
+ "capacity_gross": ("capacity_gross", "sum"),
+ },
+ )
+
+ # Merge both GDFs
+ units = pd.concat([units_with_geom, units_with_inferred_geom_gdf])
+
+ units_agg = pd.concat(
+ [
+ units_with_geom.assign(unit_count=1),
+ units_with_inferred_geom_agg_gdf,
+ ]
+ )
+ else:
+ units = units_with_geom
+ units_agg = units_with_geom.assign(unit_count=1)
+
+ # Clip to region and add mun and district ids
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+
+ write_geofile(
+ gdf=units,
+ file=snakemake.output.outfile,
+ layer_name=snakemake.config["layer"],
+ )
+ write_geofile(
+ gdf=units_agg,
+ file=snakemake.output.outfile_agg,
+ layer_name=snakemake.config["layer"],
+ )
+
+
+process()
diff --git a/digipipe/store/datasets/bnetza_mastr_pv_roof_region/config.yml b/digipipe/store/datasets/bnetza_mastr_pv_roof_region/config.yml
new file mode 100644
index 00000000..c10615d4
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_pv_roof_region/config.yml
@@ -0,0 +1,46 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Filter settings
+attributes:
+ {"EinheitMastrNummer": "mastr_id",
+ "EinheitBetriebsstatus": "status",
+ "LokationMastrNummer": "mastr_location_id",
+ "Postleitzahl": "zip_code",
+ "Ort": "city",
+ "Laengengrad": "lon",
+ "Breitengrad": "lat",
+ "Inbetriebnahmedatum": "commissioning_date",
+ "GeplantesInbetriebnahmedatum": "commissioning_date_planned",
+ "DatumEndgueltigeStilllegung": "decommissioning_date",
+ "NameStromerzeugungseinheit": "name",
+ "Bruttoleistung": "capacity_gross",
+ "Nettonennleistung": "capacity_net",
+ "Lage": "site_type",
+ "Einspeisungsart": "feedin_type",
+ "AnzahlModule": "module_count",
+ "Nutzungsbereich": "usage_sector",
+ "Hauptausrichtung": "orientation_primary",
+ "HauptausrichtungNeigungswinkel": "orientation_primary_angle",
+ "Nebenausrichtung": "orientation_secondary",
+ "NebenausrichtungNeigungswinkel": "orientation_secondary_angle",
+ "EinheitlicheAusrichtungUndNeigungswinkel": "orientation_uniform",
+ "Leistungsbegrenzung": "power_limitation",
+ "ArtDerFlaeche": "area_type",
+ "InAnspruchGenommeneFlaeche": "area_occupied",
+ "InAnspruchGenommeneAckerflaeche": "agricultural_area_occupied",
+ "Buergerenergie": "citizens_unit",
+ "MieterstromZugeordnet": "landlord_to_tenant_electricity",
+ "NetzbetreiberpruefungStatus": "validated_by_system_operator"}
+attributes_filter:
+ {"Land": "Deutschland",
+ "EinheitBetriebsstatus": ["In Betrieb", "In Planung"],
+ "Lage": [
+ "Bauliche Anlagen (Hausdach, Gebäude und Fassade)",
+ "Bauliche Anlagen (Sonstige)",
+ "Steckerfertige Erzeugungsanlage (sog. Plug-In- oder Balkon-PV-Anlage)"
+ ]}
+
+# Output settings
+layer: bnetza_mastr_pv_roof
diff --git a/digipipe/store/datasets/bnetza_mastr_pv_roof_region/create.smk b/digipipe/store/datasets/bnetza_mastr_pv_roof_region/create.smk
new file mode 100644
index 00000000..49551482
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_pv_roof_region/create.smk
@@ -0,0 +1,105 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.scripts.datasets.mastr import create_stats_per_municipality
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ PATH_TO_REGION_DISTRICTS_GPKG,
+)
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bnetza_mastr_pv_roof_region")
+SOURCE_DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "bnetza_mastr", data_dir=True
+)
+
+
+rule create:
+ """
+ Extract roof-mounted PV plants for region
+ """
+ input:
+ units=SOURCE_DATASET_PATH / "bnetza_mastr_solar_raw.csv",
+ locations=SOURCE_DATASET_PATH
+ / "bnetza_mastr_locations_extended_raw.csv",
+ gridconn=SOURCE_DATASET_PATH / "bnetza_mastr_grid_connections_raw.csv",
+ unit_correction=get_abs_dataset_path(
+ "raw", "bnetza_mastr_correction_region"
+ )
+ / "data"
+ / "bnetza_mastr_pv_roof_region_correction.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG,
+ output:
+ outfile=DATASET_PATH / "data" / "bnetza_mastr_pv_roof_region.gpkg",
+ outfile_agg=DATASET_PATH
+ / "data"
+ / "bnetza_mastr_pv_roof_agg_region.gpkg",
+ params:
+ config_file=DATASET_PATH / "config.yml",
+ script:
+ DATASET_PATH / "scripts" / "create.py"
+
+
+rule create_power_stats_muns:
+ """
+ Create stats on installed count of units and power per mun
+ """
+ input:
+ units=DATASET_PATH / "data" / "bnetza_mastr_pv_roof_region.gpkg",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ output:
+ DATASET_PATH / "data" / "bnetza_mastr_pv_roof_stats_muns.csv",
+ run:
+ units = create_stats_per_municipality(
+ units_df=gpd.read_file(input.units),
+ muns=gpd.read_file(input.region_muns),
+ column="capacity_net",
+ )
+ units["capacity_net"] = units["capacity_net"].div(1e3) # kW to MW
+ units.to_csv(output[0])
+
+
+rule create_development_over_time:
+ """
+ Create stats on development (per year) of cumulative total installed
+ capacity and cumulative number of operating units
+ """
+ input:
+ agg_region=DATASET_PATH / "data" / "bnetza_mastr_pv_roof_region.gpkg",
+ output:
+ DATASET_PATH / "data" / "bnetza_mastr_pv_roof_development_over_time.csv",
+ run:
+ df = gpd.read_file(input.agg_region)
+ df = df.loc[df["status"] == "In Betrieb"]
+ df["commissioning_date"] = pd.to_datetime(df["commissioning_date"])
+
+ df["year"] = df["commissioning_date"].dt.year
+
+ df_capacity_over_time = (
+ df.groupby("year")["capacity_net"]
+ .sum()
+ .cumsum() # Apply cumulative sum
+ .reset_index()
+ )
+
+ df_units_cumulative = (
+ df.groupby("year")
+ .agg(unit_count=("mastr_id", "count"))
+ .cumsum()
+ .reset_index()
+ )
+ df_combined = df_capacity_over_time.merge(
+ df_units_cumulative, on="year"
+ )
+ df_combined["capacity_net"] = (
+ df_combined["capacity_net"].div(1e3).round(1)
+ )
+ df_combined["year"] = df_combined["year"].astype(int)
+ df_combined.to_csv(output[0], index=False)
diff --git a/digipipe/store/datasets/bnetza_mastr_pv_roof_region/data/.gitkeep b/digipipe/store/datasets/bnetza_mastr_pv_roof_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bnetza_mastr_pv_roof_region/dataset.md b/digipipe/store/datasets/bnetza_mastr_pv_roof_region/dataset.md
new file mode 100644
index 00000000..504210f3
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_pv_roof_region/dataset.md
@@ -0,0 +1,30 @@
+# Photovoltaik-Aufdachanlagen
+
+Photovoltaik-Aufdachanlagen in der Region aus MaStR-Registerdaten als
+Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_pv_roof_stats_muns.csv`.
+
+## Datenkorrektur
+
+Einige Anlagen sind hinsichtlich Ihrer geografischen Lage oder Typs fehlerhaft.
+Anhand des Datensatzes
+[bnetza_mastr_correction_region](../../raw/bnetza_mastr_correction_region/dataset.md)
+wird für diese Anlagen eine Datenkorrektur vorgenommen.
diff --git a/digipipe/store/datasets/bnetza_mastr_pv_roof_region/scripts/create.py b/digipipe/store/datasets/bnetza_mastr_pv_roof_region/scripts/create.py
new file mode 100644
index 00000000..b395aaa3
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_pv_roof_region/scripts/create.py
@@ -0,0 +1,129 @@
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.datasets import mastr
+from digipipe.scripts.geo import (
+ overlay,
+ rename_filter_attributes,
+ write_geofile,
+)
+from digipipe.store.utils import (
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ get_names_from_nuts,
+)
+
+
+def process() -> None:
+ attrs = snakemake.config["attributes"]
+ attrs_filter = snakemake.config["attributes_filter"]
+ attrs_filter["Landkreis"] = get_names_from_nuts(
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ GLOBAL_CONFIG["global"]["geodata"]["nuts"],
+ )
+
+ units = pd.read_csv(
+ snakemake.input.units,
+ usecols=set(attrs.keys()) | set(attrs_filter.keys()),
+ dtype={"Postleitzahl": str},
+ index_col="EinheitMastrNummer",
+ )
+
+ # Apply corrections
+ unit_correction = pd.read_csv(
+ snakemake.input.unit_correction,
+ delimiter=";",
+ index_col="mastr_id",
+ )
+ units = mastr.apply_manual_corrections(
+ units_df=units,
+ units_correction_df=unit_correction,
+ )
+
+ units = rename_filter_attributes(
+ gdf=units,
+ attrs_filter_by_values=attrs_filter,
+ attrs_mapping=attrs,
+ ).set_index("mastr_id")
+
+ units = mastr.add_voltage_level(
+ units_df=units,
+ locations_path=snakemake.input.locations,
+ gridconn_path=snakemake.input.gridconn,
+ drop_location_id=False,
+ )
+
+ # Add geometry and drop units without coords and
+ # add column to indicate that location from original data was used
+ units_with_geom = mastr.add_geometry(units)
+ units_with_geom = units_with_geom.assign(
+ geometry_approximated=0,
+ )
+
+ units_without_geom = units.loc[(units.lon.isna() | units.lat.isna())].drop(
+ columns=["lon", "lat"]
+ )
+
+ # Add geometry for all units without coords (<=30 kW) and
+ # add column to indicate that location was inferred by geocoding
+ if len(units_without_geom) > 0:
+ (
+ units_with_inferred_geom_gdf,
+ units_with_inferred_geom_agg_gdf,
+ ) = mastr.geocode_units_wo_geometry(
+ units_without_geom,
+ columns_agg_functions={
+ "capacity_net": ("capacity_net", "sum"),
+ "unit_count": ("capacity_net", "count"),
+ "capacity_gross": ("capacity_gross", "sum"),
+ },
+ )
+
+ # Merge both GDFs
+ units = pd.concat([units_with_geom, units_with_inferred_geom_gdf])
+
+ units_agg = pd.concat(
+ [
+ units_with_geom.assign(unit_count=1),
+ units_with_inferred_geom_agg_gdf,
+ ]
+ )
+ else:
+ units = units_with_geom
+ units_agg = units_with_geom.assign(unit_count=1)
+
+ # Clip to region and add mun and district ids
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+
+ write_geofile(
+ gdf=units,
+ file=snakemake.output.outfile,
+ layer_name=snakemake.config["layer"],
+ )
+ write_geofile(
+ gdf=units_agg,
+ file=snakemake.output.outfile_agg,
+ layer_name=snakemake.config["layer"],
+ )
+
+
+process()
diff --git a/digipipe/store/datasets/bnetza_mastr_storage_region/config.yml b/digipipe/store/datasets/bnetza_mastr_storage_region/config.yml
new file mode 100644
index 00000000..c85268eb
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_storage_region/config.yml
@@ -0,0 +1,49 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Threshold to split between large and small battery storages in MWh
+battery_size_threshold: 0.1
+
+# Filter settings
+unit_attributes:
+ {"EinheitMastrNummer": "mastr_id",
+ "EinheitBetriebsstatus": "status",
+ "LokationMastrNummer": "mastr_location_id",
+ "Postleitzahl": "zip_code",
+ "Ort": "city",
+ "Laengengrad": "lon",
+ "Breitengrad": "lat",
+ "Inbetriebnahmedatum": "commissioning_date",
+ "GeplantesInbetriebnahmedatum": "commissioning_date_planned",
+ "DatumEndgueltigeStilllegung": "decommissioning_date",
+ "NameStromerzeugungseinheit": "name",
+ "Bruttoleistung": "capacity_gross",
+ "Nettonennleistung": "capacity_net",
+ "Einspeisungsart": "feedin_type",
+ "Einheittyp": "type",
+ "Batterietechnologie": "battery_technology",
+ "AcDcKoppelung": "ac_dc_coupling",
+ "Pumpspeichertechnologie": "pumped_hydro_technology",
+ "Notstromaggregat": "emergency_generator",
+ "NetzbetreiberpruefungStatus": "validated_by_system_operator"}
+unit_attributes_filter:
+ {"Land": "Deutschland",
+ "EinheitBetriebsstatus": ["In Betrieb", "In Planung"]}
+
+plant_attributes:
+ {"VerknuepfteEinheit": "unit_mastr_id",
+ "MastrNummer": "plant_mastr_id",
+ "NutzbareSpeicherkapazitaet": "storage_capacity"}
+
+# Output settings
+layer: bnetza_mastr_storage
+
+# Filter for PV home storages
+home_storages:
+ only_single_storages: true
+ storage_capacity_thres_max: 30
+ storage_power_thres_max: 30
+ only_single_pv_roof_units: true
+ pv_roof_capacity_thres_min: 2
+ pv_roof_capacity_thres_max: 30
diff --git a/digipipe/store/datasets/bnetza_mastr_storage_region/create.smk b/digipipe/store/datasets/bnetza_mastr_storage_region/create.smk
new file mode 100644
index 00000000..0756a2ff
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_storage_region/create.smk
@@ -0,0 +1,224 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import json
+import geopandas as gpd
+
+from digipipe.scripts.datasets.mastr import create_stats_per_municipality
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ PATH_TO_REGION_DISTRICTS_GPKG,
+)
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bnetza_mastr_storage_region")
+SOURCE_DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "bnetza_mastr", data_dir=True
+)
+
+
+rule create:
+ """
+ Extract storages for region
+ """
+ input:
+ units=SOURCE_DATASET_PATH / "bnetza_mastr_storage_raw.csv",
+ units_capacity=SOURCE_DATASET_PATH / "bnetza_mastr_storage_unit_raw.csv",
+ locations=SOURCE_DATASET_PATH
+ / "bnetza_mastr_locations_extended_raw.csv",
+ gridconn=SOURCE_DATASET_PATH / "bnetza_mastr_grid_connections_raw.csv",
+ pv_roof_units=rules.datasets_bnetza_mastr_pv_roof_region_create.output.outfile,
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG,
+ output:
+ outfile=DATASET_PATH / "data" / "bnetza_mastr_storage_region.gpkg",
+ outfile_agg=DATASET_PATH
+ / "data"
+ / "bnetza_mastr_storage_agg_region.gpkg",
+ params:
+ config_file=DATASET_PATH / "config.yml",
+ script:
+ DATASET_PATH / "scripts" / "create.py"
+
+
+rule create_power_stats_muns:
+ """
+ Create stats on installed count of units and capacity per mun
+ """
+ input:
+ units=DATASET_PATH / "data" / "bnetza_mastr_storage_region.gpkg",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ output:
+ total=DATASET_PATH / "data" / "bnetza_mastr_storage_stats_muns.csv",
+ large=DATASET_PATH
+ / "data"
+ / "bnetza_mastr_storage_large_stats_muns.csv",
+ small=DATASET_PATH
+ / "data"
+ / "bnetza_mastr_storage_small_stats_muns.csv",
+ run:
+ units = gpd.read_file(input.units)
+ units["storage_capacity"] = units["storage_capacity"].div(
+ 1e3
+ ) # kW to MW
+ muns = gpd.read_file(input.region_muns)
+ print("Battery storages:")
+
+ # All storage units
+ units_total = create_stats_per_municipality(
+ units_df=units, muns=muns, column="storage_capacity"
+ )
+ print(
+ f" Storage capacity (total): "
+ f"{units_total.storage_capacity.sum().round(3)} MWh"
+ )
+ units_total.to_csv(output.total)
+
+ # Large storage units
+ units_large = create_stats_per_municipality(
+ units_df=units.loc[
+ units.storage_capacity >= config.get("battery_size_threshold")
+ ],
+ muns=muns,
+ column="storage_capacity",
+ )
+ print(
+ f" Storage capacity (large): "
+ f"{units_large.storage_capacity.sum().round(3)} MWh"
+ )
+ units_large.to_csv(output.large)
+
+ # Small storage units
+ units_small = create_stats_per_municipality(
+ units_df=units.loc[
+ units.storage_capacity < config.get("battery_size_threshold")
+ ],
+ muns=muns,
+ column="storage_capacity",
+ )
+ print(
+ f" Storage capacity (small): "
+ f"{units_small.storage_capacity.sum().round(3)} MWh"
+ )
+ units_small.to_csv(output.small)
+
+
+rule create_storage_pv_roof_stats:
+ """
+ Calculate share of PV with storages, specific capacity and specific power of
+ storages relative to PV power.
+ """
+ input:
+ units=DATASET_PATH / "data" / "bnetza_mastr_storage_region.gpkg",
+ pv_roof_units=rules.datasets_bnetza_mastr_pv_roof_region_create.output.outfile,
+ output:
+ DATASET_PATH / "data" / "bnetza_mastr_storage_pv_roof.json",
+ run:
+ # PV home units: guess PV home systems
+ pv_roof_units = gpd.read_file(input.pv_roof_units)[
+ ["mastr_id", "capacity_net"]
+ ]
+ hs_cfg = config.get("home_storages")
+ mask = (
+ pv_roof_units.capacity_net
+ >= hs_cfg.get("pv_roof_capacity_thres_min")
+ ) & (
+ pv_roof_units.capacity_net
+ <= hs_cfg.get("pv_roof_capacity_thres_max")
+ )
+ pv_roof_units_small = pv_roof_units.loc[mask]
+
+ # Storage units: calc specific values
+ units = gpd.read_file(input.units)[
+ [
+ "capacity_net",
+ "storage_capacity",
+ "mastr_location_id",
+ "pv_roof_unit_count",
+ "pv_roof_unit_capacity_sum",
+ ]
+ ]
+ units_unique_loc = units.groupby("mastr_location_id").agg(
+ storage_count=("storage_capacity", "count"),
+ storage_capacity=("storage_capacity", "sum"),
+ capacity_net=("capacity_net", "mean"),
+ pv_roof_unit_count=("pv_roof_unit_count", "first"),
+ pv_roof_unit_capacity_sum=("pv_roof_unit_capacity_sum", "first"),
+ )
+
+ # Filter storages
+ mask = (
+ (
+ units_unique_loc.storage_count == 1
+ if hs_cfg.get("only_single_storages")
+ else units_unique_loc.storage_count > 0
+ )
+ & (
+ units_unique_loc.storage_capacity
+ <= hs_cfg.get("storage_capacity_thres_max")
+ )
+ & (
+ units_unique_loc.capacity_net
+ <= hs_cfg.get("storage_power_thres_max")
+ )
+ & (
+ units_unique_loc.pv_roof_unit_count == 1
+ if hs_cfg.get("only_single_pv_roof_units")
+ else units_unique_loc.pv_roof_unit_count > 0
+ )
+ & (
+ units_unique_loc.pv_roof_unit_capacity_sum
+ >= hs_cfg.get("pv_roof_capacity_thres_min")
+ )
+ & (
+ units_unique_loc.pv_roof_unit_capacity_sum
+ <= hs_cfg.get("pv_roof_capacity_thres_max")
+ )
+ )
+ units_unique_loc_small = units_unique_loc.loc[mask]
+
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(
+ {
+ "pv_roof_share": {
+ "all_storages": round(
+ len(units_unique_loc) / len(pv_roof_units),
+ 3,
+ ),
+ "home_storages": round(
+ len(units_unique_loc_small)
+ / len(pv_roof_units_small),
+ 3,
+ ),
+ },
+ "specific_capacity": {
+ "all_storages": round(
+ units_unique_loc.storage_capacity.sum()
+ / units_unique_loc.pv_roof_unit_capacity_sum.sum(),
+ 2,
+ ),
+ "home_storages": round(
+ units_unique_loc_small.storage_capacity.sum()
+ / units_unique_loc_small.pv_roof_unit_capacity_sum.sum(),
+ 2,
+ ),
+ },
+ "specific_power": {
+ "all_storages": round(
+ units_unique_loc.capacity_net.sum()
+ / units_unique_loc.pv_roof_unit_capacity_sum.sum(),
+ 2,
+ ),
+ "home_storages": round(
+ units_unique_loc_small.capacity_net.sum()
+ / units_unique_loc_small.pv_roof_unit_capacity_sum.sum(),
+ 2,
+ ),
+ },
+ },
+ f,
+ indent=4,
+ )
diff --git a/digipipe/store/datasets/bnetza_mastr_storage_region/data/.gitkeep b/digipipe/store/datasets/bnetza_mastr_storage_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bnetza_mastr_storage_region/dataset.md b/digipipe/store/datasets/bnetza_mastr_storage_region/dataset.md
new file mode 100644
index 00000000..06469720
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_storage_region/dataset.md
@@ -0,0 +1,38 @@
+# Speicheranlagen
+
+Speicheranlagen in der Region aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Es wird weiterhin geprüft, ob dem Speicher eine oder mehrere PV-Aufdachanlagen
+zugeordnet sind, es wird die Anzahl und Summe der Nettonennleistung berechnet.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Weiterhin erfolgt eine Auswertung der installierten Gesamtleistung je Gemeinde:
+
+- Alle Speicher: `bnetza_mastr_storage_stats_muns.csv`
+- Großspeicher (>=100 kWh): `bnetza_mastr_storage_large_stats_muns.csv`
+- Kleinspeicher (<100 kWh): `bnetza_mastr_storage_small_stats_muns.csv`
+
+`bnetza_mastr_storage_pv_roof.json` enthält die spezifische Speicherkapazität
+sowie spezifische Nennleistung der Speicher (bezogen auf die installierte
+Leistung von PV-Aufdachanlagen), aggregiert für gesamte Region, für folgende
+Randbedingungen:
+
+- Alle PV-Anlagen: `all_storages`
+- PV-Anlagen mit 2..20 kWp sowie Batteriespeicher <20 kWh und <20 kW (kann in
+ [config.yml](config.yml) unter `home_storages` konfiguriert werden):
+ `home_storages`
diff --git a/digipipe/store/datasets/bnetza_mastr_storage_region/scripts/create.py b/digipipe/store/datasets/bnetza_mastr_storage_region/scripts/create.py
new file mode 100644
index 00000000..ba70600b
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_storage_region/scripts/create.py
@@ -0,0 +1,166 @@
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.datasets import mastr
+from digipipe.scripts.geo import (
+ overlay,
+ rename_filter_attributes,
+ write_geofile,
+)
+from digipipe.store.utils import (
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ get_names_from_nuts,
+)
+
+
+def process() -> None:
+ unit_attrs = snakemake.config["unit_attributes"]
+ unit_attrs_filter = snakemake.config["unit_attributes_filter"]
+ unit_attrs_filter["Landkreis"] = get_names_from_nuts(
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ GLOBAL_CONFIG["global"]["geodata"]["nuts"],
+ )
+ # Read units
+ units = pd.read_csv(
+ snakemake.input.units,
+ usecols=(set(unit_attrs.keys()) | set(unit_attrs_filter.keys())),
+ dtype={"Postleitzahl": str},
+ )
+ units = rename_filter_attributes(
+ gdf=units,
+ attrs_filter_by_values=unit_attrs_filter,
+ attrs_mapping=unit_attrs,
+ ).set_index("mastr_id")
+
+ # Read plants (for storage capacity)
+ plants = pd.read_csv(
+ snakemake.input.units_capacity,
+ usecols=snakemake.config["plant_attributes"].keys(),
+ )
+ plants = rename_filter_attributes(
+ gdf=plants,
+ attrs_mapping=snakemake.config["plant_attributes"],
+ )
+
+ # Merge storage capacity
+ units = units.merge(
+ plants,
+ left_on="mastr_id",
+ right_on="unit_mastr_id",
+ how="left",
+ )
+ units_count_wo_capacity = len(units.loc[units.plant_mastr_id.isna()])
+ if units_count_wo_capacity > 0:
+ print(
+ f"{units_count_wo_capacity} storages have no plant associated and "
+ f"hence no storage capacity assigned."
+ )
+ units.drop(columns=["unit_mastr_id", "plant_mastr_id"], inplace=True)
+
+ units = mastr.add_voltage_level(
+ units_df=units,
+ locations_path=snakemake.input.locations,
+ gridconn_path=snakemake.input.gridconn,
+ drop_location_id=False,
+ )
+
+ # Add geometry and drop units without coords and
+ # add column to indicate that location from original data was used
+ units_with_geom = mastr.add_geometry(units)
+ units_with_geom = units_with_geom.assign(
+ geometry_approximated=0,
+ )
+
+ units_without_geom = units.loc[(units.lon.isna() | units.lat.isna())].drop(
+ columns=["lon", "lat"]
+ )
+
+ # Add geometry for all units without coords (<=30 kW) and
+ # add column to indicate that location was inferred by geocoding
+ if len(units_without_geom) > 0:
+ (
+ units_with_inferred_geom_gdf,
+ units_with_inferred_geom_agg_gdf,
+ ) = mastr.geocode_units_wo_geometry(
+ units_without_geom,
+ columns_agg_functions={
+ "capacity_net": ("capacity_net", "sum"),
+ "unit_count": ("capacity_net", "count"),
+ "capacity_gross": ("capacity_gross", "sum"),
+ "storage_capacity": ("storage_capacity", "sum"),
+ },
+ )
+
+ # Merge both GDFs
+ units = pd.concat([units_with_geom, units_with_inferred_geom_gdf])
+
+ units_agg = pd.concat(
+ [
+ units_with_geom.assign(unit_count=1),
+ units_with_inferred_geom_agg_gdf,
+ ]
+ )
+ else:
+ units = units_with_geom
+ units_agg = units_with_geom.assign(unit_count=1)
+
+ # Clip to region and add mun and district ids
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+
+ # Search for associated PV roof units and save count and sum of nom. power
+ pv_roof_units = gpd.read_file(snakemake.input.pv_roof_units)[
+ ["mastr_location_id", "capacity_net"]
+ ]
+ pv_roof_units = (
+ pv_roof_units[["mastr_location_id", "capacity_net"]]
+ .groupby("mastr_location_id")
+ .agg(
+ pv_roof_unit_count=("capacity_net", "count"),
+ pv_roof_unit_capacity_sum=("capacity_net", "sum"),
+ )
+ .reset_index()
+ )
+ units = units.merge(
+ pv_roof_units,
+ left_on="mastr_location_id",
+ right_on="mastr_location_id",
+ how="left",
+ )
+ units = units.assign(
+ pv_roof_unit_count=units.pv_roof_unit_count.fillna(0),
+ pv_roof_unit_capacity_sum=units.pv_roof_unit_capacity_sum.fillna(0),
+ )
+
+ write_geofile(
+ gdf=units,
+ file=snakemake.output.outfile,
+ layer_name=snakemake.config["layer"],
+ )
+ write_geofile(
+ gdf=units_agg,
+ file=snakemake.output.outfile_agg,
+ layer_name=snakemake.config["layer"],
+ )
+
+
+process()
diff --git a/digipipe/store/datasets/bnetza_mastr_wind_region/config.yml b/digipipe/store/datasets/bnetza_mastr_wind_region/config.yml
new file mode 100644
index 00000000..60fbb8de
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_wind_region/config.yml
@@ -0,0 +1,40 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Filter settings
+attributes:
+ {"EinheitMastrNummer": "mastr_id",
+ "EinheitBetriebsstatus": "status",
+ "LokationMastrNummer": "mastr_location_id",
+ "Postleitzahl": "zip_code",
+ "Ort": "city",
+ "Laengengrad": "lon",
+ "Breitengrad": "lat",
+ "Inbetriebnahmedatum": "commissioning_date",
+ "GeplantesInbetriebnahmedatum": "commissioning_date_planned",
+ "DatumEndgueltigeStilllegung": "decommissioning_date",
+ "NameStromerzeugungseinheit": "name",
+ "NameWindpark": "name_park",
+ "Bruttoleistung": "capacity_gross",
+ "Nettonennleistung": "capacity_net",
+ "Lage": "site_type",
+ "Hersteller": "manufacturer_name",
+ "Typenbezeichnung": "type_name",
+ "Nabenhoehe": "hub_height",
+ "Rotordurchmesser": "rotor_diameter",
+ "AuflageAbschaltungLeistungsbegrenzung": "constraint_deactivation_sound_emission",
+ "AuflagenAbschaltungSchallimmissionsschutzNachts": "constraint_deactivation_sound_emission_night",
+ "AuflagenAbschaltungSchallimmissionsschutzTagsueber": "constraint_deactivation_sound_emission_day",
+ "AuflagenAbschaltungSchattenwurf": "constraint_deactivation_shadowing",
+ "AuflagenAbschaltungTierschutz": "constraint_deactivation_animals",
+ "AuflagenAbschaltungEiswurf": "constraint_deactivation_ice",
+ "Buergerenergie": "citizens_unit",
+ "NetzbetreiberpruefungStatus": "validated_by_system_operator"}
+attributes_filter:
+ {"Land": "Deutschland",
+ "EinheitBetriebsstatus": ["In Betrieb", "In Planung"],
+ "Lage": "Windkraft an Land"}
+
+# Output settings
+layer: bnetza_mastr_wind
diff --git a/digipipe/store/datasets/bnetza_mastr_wind_region/create.smk b/digipipe/store/datasets/bnetza_mastr_wind_region/create.smk
new file mode 100644
index 00000000..de9b6e21
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_wind_region/create.smk
@@ -0,0 +1,98 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.scripts.datasets.mastr import create_stats_per_municipality
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ PATH_TO_REGION_DISTRICTS_GPKG,
+)
+
+DATASET_PATH = get_abs_dataset_path("datasets", "bnetza_mastr_wind_region")
+SOURCE_DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "bnetza_mastr", data_dir=True
+)
+
+
+rule create:
+ """
+ Extract wind turbines for region
+ """
+ input:
+ units=SOURCE_DATASET_PATH / "bnetza_mastr_wind_raw.csv",
+ locations=SOURCE_DATASET_PATH
+ / "bnetza_mastr_locations_extended_raw.csv",
+ gridconn=SOURCE_DATASET_PATH / "bnetza_mastr_grid_connections_raw.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG,
+ output:
+ outfile=DATASET_PATH / "data" / "bnetza_mastr_wind_region.gpkg",
+ outfile_agg=DATASET_PATH / "data" / "bnetza_mastr_wind_agg_region.gpkg",
+ params:
+ config_file=DATASET_PATH / "config.yml",
+ script:
+ DATASET_PATH / "scripts" / "create.py"
+
+
+rule create_power_stats_muns:
+ """
+ Create stats on installed count of units and power per mun
+ """
+ input:
+ units=DATASET_PATH / "data" / "bnetza_mastr_wind_region.gpkg",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ output:
+ DATASET_PATH / "data" / "bnetza_mastr_wind_stats_muns.csv",
+ run:
+ units = create_stats_per_municipality(
+ units_df=gpd.read_file(input.units),
+ muns=gpd.read_file(input.region_muns),
+ column="capacity_net",
+ )
+ units["capacity_net"] = units["capacity_net"].div(1e3) # kW to MW
+ units.to_csv(output[0])
+
+
+rule create_development_over_time:
+ """
+ Create stats on development (per year) of cumulative total installed
+ capacity and cumulative number of operating units
+ """
+ input:
+ agg_region=DATASET_PATH / "data" / "bnetza_mastr_wind_region.gpkg",
+ output:
+ DATASET_PATH / "data" / "bnetza_mastr_wind_development_over_time.csv",
+ run:
+ df = gpd.read_file(input.agg_region)
+ df = df.loc[df["status"] == "In Betrieb"]
+ df["commissioning_date"] = pd.to_datetime(df["commissioning_date"])
+
+ df["year"] = df["commissioning_date"].dt.year
+
+ df_capacity_over_time = (
+ df.groupby("year")["capacity_net"]
+ .sum()
+ .cumsum() # Apply cumulative sum
+ .reset_index()
+ )
+
+ df_units_cumulative = (
+ df.groupby("year")
+ .agg(unit_count=("mastr_id", "count"))
+ .cumsum()
+ .reset_index()
+ )
+ df_combined = df_capacity_over_time.merge(
+ df_units_cumulative, on="year"
+ )
+ df_combined["capacity_net"] = (
+ df_combined["capacity_net"].div(1e3).round(1)
+ )
+ df_combined["year"] = df_combined["year"].astype(int)
+ df_combined.to_csv(output[0], index=False)
diff --git a/digipipe/store/datasets/bnetza_mastr_wind_region/data/.gitkeep b/digipipe/store/datasets/bnetza_mastr_wind_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/bnetza_mastr_wind_region/dataset.md b/digipipe/store/datasets/bnetza_mastr_wind_region/dataset.md
new file mode 100644
index 00000000..732e6900
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_wind_region/dataset.md
@@ -0,0 +1,22 @@
+# Windenergieanlagen
+
+Windenergieanlagen in der Region aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_wind_stats_muns.csv`.
diff --git a/digipipe/store/datasets/bnetza_mastr_wind_region/scripts/create.py b/digipipe/store/datasets/bnetza_mastr_wind_region/scripts/create.py
new file mode 100644
index 00000000..c2314120
--- /dev/null
+++ b/digipipe/store/datasets/bnetza_mastr_wind_region/scripts/create.py
@@ -0,0 +1,116 @@
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.datasets import mastr
+from digipipe.scripts.geo import (
+ overlay,
+ rename_filter_attributes,
+ write_geofile,
+)
+from digipipe.store.utils import (
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ get_names_from_nuts,
+)
+
+
+def process() -> None:
+ attrs = snakemake.config["attributes"]
+ attrs_filter = snakemake.config["attributes_filter"]
+ attrs_filter["Landkreis"] = get_names_from_nuts(
+ PATH_TO_REGION_DISTRICTS_GPKG,
+ GLOBAL_CONFIG["global"]["geodata"]["nuts"],
+ )
+
+ units = pd.read_csv(
+ snakemake.input.units,
+ usecols=set(attrs.keys()) | set(attrs_filter.keys()),
+ dtype={"Postleitzahl": str},
+ )
+
+ units = rename_filter_attributes(
+ gdf=units,
+ attrs_filter_by_values=attrs_filter,
+ attrs_mapping=attrs,
+ ).set_index("mastr_id")
+
+ units = mastr.add_voltage_level(
+ units_df=units,
+ locations_path=snakemake.input.locations,
+ gridconn_path=snakemake.input.gridconn,
+ )
+
+ # Add geometry and drop units without coords and
+ # add column to indicate that location from original data was used
+ units_with_geom = mastr.add_geometry(units)
+ units_with_geom = units_with_geom.assign(
+ geometry_approximated=0,
+ )
+
+ units_without_geom = units.loc[(units.lon.isna() | units.lat.isna())].drop(
+ columns=["lon", "lat"]
+ )
+
+ # Add geometry for all units without coords (<=30 kW) and
+ # add column to indicate that location was inferred by geocoding
+ if len(units_without_geom) > 0:
+ (
+ units_with_inferred_geom_gdf,
+ units_with_inferred_geom_agg_gdf,
+ ) = mastr.geocode_units_wo_geometry(
+ units_without_geom,
+ columns_agg_functions={
+ "capacity_net": ("capacity_net", "sum"),
+ "unit_count": ("capacity_net", "count"),
+ "capacity_gross": ("capacity_gross", "sum"),
+ },
+ )
+
+ # Merge both GDFs
+ units = pd.concat([units_with_geom, units_with_inferred_geom_gdf])
+
+ units_agg = pd.concat(
+ [
+ units_with_geom.assign(unit_count=1),
+ units_with_inferred_geom_agg_gdf,
+ ]
+ )
+ else:
+ units = units_with_geom
+ units_agg = units_with_geom.assign(unit_count=1)
+
+ # Clip to region and add mun and district ids
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units = overlay(
+ gdf=units,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ units_agg = overlay(
+ gdf=units_agg,
+ gdf_overlay=gpd.read_file(snakemake.input.region_districts),
+ retain_rename_overlay_columns={"id": "district_id"},
+ )
+
+ write_geofile(
+ gdf=units,
+ file=snakemake.output.outfile,
+ layer_name=snakemake.config["layer"],
+ )
+ write_geofile(
+ gdf=units_agg,
+ file=snakemake.output.outfile_agg,
+ layer_name=snakemake.config["layer"],
+ )
+
+
+process()
diff --git a/digipipe/store/datasets/demand_electricity_region/config.yml b/digipipe/store/datasets/demand_electricity_region/config.yml
new file mode 100644
index 00000000..c8934a8c
--- /dev/null
+++ b/digipipe/store/datasets/demand_electricity_region/config.yml
@@ -0,0 +1,34 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Household demand: years to create demand prognoses for
+hh_electricity_demand:
+ years:
+ [
+ 2022,
+ #2035,
+ 2045
+ ]
+
+# CTS demand: years to create demand prognoses for
+cts_electricity_demand:
+ years:
+ [
+ 2022,
+ #2035,
+ 2045
+ ]
+
+# Industry demand: years to create demand prognoses for
+ind_electricity_demand:
+ years:
+ [
+ 2022,
+ #2035,
+ 2045
+ ]
+
+# Industry demand: dataset to be used for today's industrial power consumption,
+# must be one of ["demandregio", "stala"]
+ind_electricity_demand_source: "stala"
diff --git a/digipipe/store/datasets/demand_electricity_region/create.smk b/digipipe/store/datasets/demand_electricity_region/create.smk
new file mode 100644
index 00000000..bfb8083f
--- /dev/null
+++ b/digipipe/store/datasets/demand_electricity_region/create.smk
@@ -0,0 +1,302 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import difflib
+import geopandas as gpd
+import pandas as pd
+
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ PATH_TO_REGION_DISTRICTS_GPKG
+)
+from digipipe.scripts.datasets.demand import (
+ demand_prognosis,
+ disaggregate_demand_to_municipality,
+ merge_demand_multiple_years,
+ normalize_filter_timeseries
+)
+
+DATASET_PATH = get_abs_dataset_path("datasets", "demand_electricity_region")
+
+rule hh_normalize_timeseries:
+ """
+ Extract household electricity demand timeseries for districts, merge and
+ normalize them
+ """
+ input:
+ timeseries=get_abs_dataset_path("preprocessed", "demandregio") /
+ "data" / "dr_hh_power_timeseries_2022.csv",
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG
+ output:
+ timeseries=DATASET_PATH / "data" / "demand_hh_power_timeseries.csv"
+ run:
+ normalize_filter_timeseries(
+ infile=input.timeseries,
+ outfile=output.timeseries,
+ region_nuts=gpd.read_file(input.region_districts).nuts.to_list(),
+ )
+
+rule hh_disaggregate_demand:
+ """
+ Disaggregate household electricity demand from districts to
+ municipalities for one year and create prognosis
+ """
+ input:
+ demand_today_region=get_abs_dataset_path("preprocessed", "demandregio") /
+ "data" / "dr_hh_power_demand_2022.csv",
+ demand_future_TN=get_abs_dataset_path(
+ "preprocessed","bmwk_long_term_scenarios"
+ ) / "data" / "TN-Strom_hh_demand_reformatted.csv",
+ demand_future_T45=get_abs_dataset_path(
+ "preprocessed","bmwk_long_term_scenarios"
+ ) / "data" / "T45-Strom_hh_demand_reformatted.csv",
+ population=get_abs_dataset_path("datasets", "population_region") /
+ "data" / "population.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG
+ output:
+ demand=DATASET_PATH / "data" / "demand_hh_power_demand_{year}.csv"
+ run:
+ population = pd.read_csv(input.population, header=[0, 1], index_col=0)
+ population.columns = population.columns.droplevel(1)
+ # Today's demand
+ demand_districts = pd.read_csv(
+ input.demand_today_region
+ ).set_index("nuts3").sum(axis=1).to_frame(name="demand_districts") * 1e3
+ demand = disaggregate_demand_to_municipality(
+ demand_districts=demand_districts,
+ muns=gpd.read_file(input.region_muns),
+ districts=gpd.read_file(input.region_districts),
+ disagg_data=population,
+ disagg_data_col=str(wildcards.year),
+ )
+ # Future demand
+ if int(wildcards.year) > 2022:
+ demand = demand_prognosis(
+ demand_future_T45=input.demand_future_T45,
+ demand_future_TN=input.demand_future_TN,
+ demand_region=demand,
+ year_base=2022,
+ year_target=int(wildcards.year),
+ scale_by="carrier",
+ carrier="electricity",
+ )
+ demand.to_csv(output.demand)
+
+rule hh_merge_demand_years:
+ """
+ Merge the electricity demands from different years into one
+ """
+ input:
+ demand=expand(
+ DATASET_PATH / "data" / "demand_hh_power_demand_{year}.csv",
+ year=config["hh_electricity_demand"]["years"]
+ )
+ output:
+ demand=DATASET_PATH / "data" / "demand_hh_power_demand.csv"
+ run:
+ merge_demand_multiple_years(
+ infiles=input.demand,
+ outfile=output.demand,
+ )
+
+rule cts_normalize_timeseries:
+ """
+ Extract CTS electricity demand timeseries for districts, merge and
+ normalize them
+ """
+ input:
+ timeseries=get_abs_dataset_path("preprocessed", "demandregio") /
+ "data" / "dr_cts_power_timeseries_2022.csv",
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG
+ output:
+ timeseries=DATASET_PATH / "data" / "demand_cts_power_timeseries.csv"
+ run:
+ normalize_filter_timeseries(
+ infile=input.timeseries,
+ outfile=output.timeseries,
+ region_nuts=gpd.read_file(input.region_districts).nuts.to_list(),
+ )
+
+rule cts_disaggregate_demand:
+ """
+ Disaggregate CTS electricity demand from districts to municipalities
+ for one year
+ """
+ input:
+ demand_today_region=get_abs_dataset_path(
+ "preprocessed", "demandregio") / "data" /
+ "dr_cts_power_demand_2022.csv",
+ demand_future_TN=get_abs_dataset_path(
+ "preprocessed", "bmwk_long_term_scenarios"
+ ) / "data" / "TN-Strom_cts_demand_reformatted.csv",
+ demand_future_T45=get_abs_dataset_path(
+ "preprocessed", "bmwk_long_term_scenarios"
+ ) / "data" / "T45-Strom_cts_demand_reformatted.csv",
+ employment=get_abs_dataset_path("datasets", "employment_region") /
+ "data" / "employment.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG
+ output:
+ demand=DATASET_PATH / "data" / "demand_cts_power_demand_{year}.csv"
+ run:
+ # Today's demand
+ demand_districts = pd.read_csv(
+ input.demand_today_region,
+ index_col=0
+ ).sum(axis=0).T.to_frame(name="demand_districts")
+ demand = disaggregate_demand_to_municipality(
+ demand_districts=demand_districts,
+ muns=gpd.read_file(input.region_muns),
+ districts=gpd.read_file(input.region_districts),
+ disagg_data=pd.read_csv(
+ input.employment,
+ index_col=0,
+ ),
+ disagg_data_col="employees_total"
+ )
+ # Future demand
+ if int(wildcards.year) > 2022:
+ demand = demand_prognosis(
+ demand_future_T45=input.demand_future_T45,
+ demand_future_TN=input.demand_future_TN,
+ demand_region=demand,
+ year_base=2022,
+ year_target=int(wildcards.year),
+ scale_by="carrier",
+ carrier="electricity",
+ )
+ demand.rename(columns={"employees_total": wildcards.year}).to_csv(output.demand)
+
+rule cts_merge_demand_years:
+ """
+ Merge the electricity demands from different years into one
+ """
+ input:
+ demand=expand(
+ DATASET_PATH / "data" / "demand_cts_power_demand_{year}.csv",
+ year=config["cts_electricity_demand"]["years"]
+ )
+ output:
+ demand=DATASET_PATH / "data" / "demand_cts_power_demand.csv"
+ run:
+ merge_demand_multiple_years(
+ infiles=input.demand,
+ outfile=output.demand,
+ )
+
+rule ind_normalize_timeseries:
+ """
+ Extract industry electricity demand timeseries for districts, merge and
+ normalize them
+ """
+ input:
+ timeseries=get_abs_dataset_path("preprocessed", "demandregio") /
+ "data" / "dr_ind_power_timeseries_2022.csv",
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG
+ output:
+ timeseries=DATASET_PATH / "data" / "demand_ind_power_timeseries.csv"
+ run:
+ normalize_filter_timeseries(
+ infile=input.timeseries,
+ outfile=output.timeseries,
+ region_nuts=gpd.read_file(input.region_districts).nuts.to_list(),
+ )
+
+rule ind_disaggregate_demand:
+ """
+ Disaggregate industry electricity demand from districts to
+ municipalities for one year
+ """
+ input:
+ demand_today_region_dr=get_abs_dataset_path(
+ "preprocessed", "demandregio") /
+ "data" / "dr_ind_power_demand_2022.csv",
+ demand_today_region_stala=get_abs_dataset_path(
+ "preprocessed", "stala_st_energy") /
+ "data" / "power_demand_industry_st_districts.csv",
+ demand_future_TN=get_abs_dataset_path(
+ "preprocessed","bmwk_long_term_scenarios"
+ ) / "data" / "TN-Strom_ind_demand_reformatted.csv",
+ demand_future_T45=get_abs_dataset_path(
+ "preprocessed","bmwk_long_term_scenarios"
+ ) / "data" / "T45-Strom_ind_demand_reformatted.csv",
+ employment=get_abs_dataset_path("datasets", "employment_region") /
+ "data" / "employment.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG
+ output:
+ demand=DATASET_PATH / "data" / "demand_ind_power_demand_{year}.csv"
+ run:
+ districts = gpd.read_file(input.region_districts)
+ # Today's demand: use dataset defined in config
+ if config["ind_electricity_demand_source"] == "demandregio":
+ demand_districts = pd.read_csv(
+ input.demand_today_region_dr,
+ index_col=0
+ ).sum(axis=0).T.to_frame(name="demand_districts")
+ elif config["ind_electricity_demand_source"] == "stala":
+ demand_districts = pd.read_csv(
+ input.demand_today_region_stala,
+ usecols=["name", "2021"],
+ index_col="name"
+ )
+ districts["name"] = districts["name"].map(
+ lambda _: difflib.get_close_matches(
+ _, demand_districts.index)[0]
+ )
+ demand_districts = demand_districts.merge(districts,on="name")[
+ ["nuts", "2021"]].rename(columns={
+ "2021": "demand_districts"}).set_index("nuts")
+ else:
+ raise ValueError(
+ "ind_electricity_demand_source must be one of "
+ "['demandregio', 'stala']"
+ )
+
+ # Disaggregate
+ demand = disaggregate_demand_to_municipality(
+ demand_districts=demand_districts,
+ muns=gpd.read_file(input.region_muns),
+ districts=districts,
+ disagg_data=pd.read_csv(
+ input.employment,
+ index_col=0,
+ ),
+ disagg_data_col="employees_ind"
+ )
+
+ # Future demand
+ if int(wildcards.year) > 2022:
+ demand = demand_prognosis(
+ demand_future_T45=input.demand_future_T45,
+ demand_future_TN=input.demand_future_TN,
+ demand_region=demand,
+ year_base=2022,
+ year_target=int(wildcards.year),
+ scale_by="total",
+ carrier="electricity",
+ )
+ demand.rename(
+ columns={"employees_ind": wildcards.year}
+ ).to_csv(output.demand)
+
+rule ind_merge_demand_years:
+ """
+ Merge the electricity demands from different years into one
+ """
+ input:
+ demand=expand(
+ DATASET_PATH / "data" / "demand_ind_power_demand_{year}.csv",
+ year=config["ind_electricity_demand"]["years"]
+ )
+ output:
+ demand=DATASET_PATH / "data" / "demand_ind_power_demand.csv"
+ run:
+ merge_demand_multiple_years(
+ infiles=input.demand,
+ outfile=output.demand,
+ )
diff --git a/digipipe/store/datasets/demand_electricity_region/data/.gitkeep b/digipipe/store/datasets/demand_electricity_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/demand_electricity_region/dataset.md b/digipipe/store/datasets/demand_electricity_region/dataset.md
new file mode 100644
index 00000000..de221aa3
--- /dev/null
+++ b/digipipe/store/datasets/demand_electricity_region/dataset.md
@@ -0,0 +1,76 @@
+# Strombedarf
+
+Nettostrombedarfe und -zeitreihen für Haushalte, GHD und Industrie je Gemeinde.
+
+Die Berechnung der regionalen Prognosewerte je Verbrauchssektor erfolgt anhand
+landesweiter Prognosen aus den
+[BMWK Langfristszenarien](../../preprocessed/bmwk_long_term_scenarios/dataset.md).
+
+## Haushalte
+
+- Jährlicher Strombedarf je Gemeinde in MWh aus
+ [DemandRegio](../../preprocessed/demandregio/dataset.md), von Landkreis- auf
+ Gemeindeebene disaggregiert anhand von Bevölkerungsprognosen
+ ([STALA ST](../../preprocessed/stala_st_pop_prog/dataset.md)).
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion des
+ Strombedarfs (ohne Wärmegewinnung) aus
+ [BMWK Langfristszenarien](../../preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Hierbei wird das Szenario "TN-Strom" als Grundlage für den Status
+ quo verwendet und Werte für 2022 interpoliert. Die Zielwerte werden dem
+ Szenario "T45-Strom" entnommen.
+- Gemittelte, normierte Strombedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../preprocessed/demandregio/dataset.md)-Daten von 2022, die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird, da die Basis
+ SLP-Profile sind und Differenzen zwischen verschiedenen Jahren nur aufgrund
+ der Lage von Wochenenden und Feiertagen bestehen. Diese werden daher
+ vernachlässigt.
+
+## GHD
+
+- Jährlicher Strombedarf je Gemeinde in MWh aus
+ [DemandRegio](../../preprocessed/demandregio/dataset.md), von Landkreis- auf
+ Gemeindeebene disaggregiert anhand von sozialversicherungspflichtig
+ Beschäftigten im Jahr 2022
+ ([BA für Arbeit](../../preprocessed/ba_employment/dataset.md)).
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion des
+ Strombedarfs (ohne Wärmegewinnung) aus
+ [BMWK Langfristszenarien](../../preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Hierbei wird das Szenario "TN-Strom" als Grundlage für den Status
+ quo verwendet und Werte für 2022 interpoliert. Die Zielwerte werden dem
+ Szenario "T45-Strom" entnommen.
+- Gemittelte, normierte Strombedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../preprocessed/demandregio/dataset.md)-Daten von 2022, die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird. Basis bilden
+ sowohl SLP- als auch branchenspezifische Profile. Aufgrund der geringen
+ Differenzen zwischen den Landkreisen werden diese gemittelt. Differenzen
+ zwischen verschiedenen Jahren bestehen nur aufgrund der Lage von Wochenenden
+ und Feiertagen und werden daher vernachlässigt.
+
+## Industrie
+
+- Jährlicher Strombedarf je Gemeinde in MWh. Hierfür stehen 2 Datensätze zur
+ Verfügung - welcher verwendet wird, kann in der [Konfiguration](config.yml)
+ via `ind_electricity_demand_source` eingestellt werden:
+ - [DemandRegio](../../preprocessed/demandregio/dataset.md): Werte für alle
+ Landkreise in Deutschland.
+ - [STALA ST](../../preprocessed/stala_st_energy/dataset.md) (Standard):
+ Genauere Werte, jedoch nur für Sachsen-Anhalt verfügbar.
+- Die Desaggregation von Landkreis- auf Gemeindeebene erfolgt anhand der
+ Beschäftigten im verarbeitenden Gewerbe im Jahr 2022
+ ([Regionalstatistik](../../preprocessed/regiostat/dataset.md)).
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion des
+ industriellen Gesamtenergiebedarfs aus
+ [BMWK Langfristszenarien](../../preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Im Unterschied zu Haushalten und GHD liegen die Daten für den
+ Wärme- und Stromanteil nicht getrennt vor, sodass auf den
+ Gesamtenergiebedarf zurückgegriffen wird.
+ Es wird das Szenario "TN-Strom" als Grundlage für den Status quo verwendet und
+ Werte für 2022 interpoliert. Die Zielwerte werden dem Szenario "T45-Strom"
+ entnommen.
+- Gemittelte, normierte Strombedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../preprocessed/demandregio/dataset.md)-Daten von 2022, die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird. Basis bilden
+ sowohl SLP- als auch branchenspezifische Profile. Aufgrund der geringen
+ Differenzen zwischen den Landkreisen werden diese gemittelt. Differenzen
+ zwischen verschiedenen Jahren bestehen nur aufgrund der Lage von Wochenenden
+ und Feiertagen und werden daher vernachlässigt.
diff --git a/digipipe/store/datasets/demand_electricity_region/scripts/create.py b/digipipe/store/datasets/demand_electricity_region/scripts/create.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/demand_heat_region/__init__.py b/digipipe/store/datasets/demand_heat_region/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/demand_heat_region/config.yml b/digipipe/store/datasets/demand_heat_region/config.yml
new file mode 100644
index 00000000..8a8502da
--- /dev/null
+++ b/digipipe/store/datasets/demand_heat_region/config.yml
@@ -0,0 +1,103 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Share of heat demand in district heating networks of total heat demand
+# Format: municipality_id: share
+district_heating_share:
+ # Households
+ hh:
+ {0: 0.36, # Dessau-Roßlau
+ 1: 0,
+ 2: 0.11, # Bitterfeld-Wolfen
+ 3: 0.07, # Köthen
+ 4: 0,
+ 5: 0,
+ 6: 0,
+ 7: 0,
+ 8: 0,
+ 9: 0,
+ 10: 0,
+ 11: 0,
+ 12: 0,
+ 13: 0,
+ 14: 0,
+ 15: 0,
+ 16: 0,
+ 17: 0,
+ 18: 0.15, # Wittenberg
+ 19: 0}
+ # CTS
+ cts:
+ {0: 0.36, # Dessau-Roßlau
+ 1: 0,
+ 2: 0.11, # Bitterfeld-Wolfen
+ 3: 0.07, # Köthen
+ 4: 0,
+ 5: 0,
+ 6: 0,
+ 7: 0,
+ 8: 0,
+ 9: 0,
+ 10: 0,
+ 11: 0,
+ 12: 0,
+ 13: 0,
+ 14: 0,
+ 15: 0,
+ 16: 0,
+ 17: 0,
+ 18: 0.15, # Wittenberg
+ 19: 0}
+ # industry
+ ind:
+ {0: 0.19, # Dessau-Roßlau
+ 1: 0,
+ 2: 0.21, # Bitterfeld-Wolfen
+ 3: 0.21, # Köthen
+ 4: 0,
+ 5: 0,
+ 6: 0,
+ 7: 0,
+ 8: 0,
+ 9: 0,
+ 10: 0,
+ 11: 0,
+ 12: 0,
+ 13: 0,
+ 14: 0,
+ 15: 0,
+ 16: 0,
+ 17: 0,
+ 18: 0.01, # Wittenberg
+ 19: 0}
+
+# Heating structure
+heating_structure:
+ years:
+ [2022, 2045]
+
+# Aggregation of carriers used in technologies generating central heat
+# in 2022 and 2045
+carrier_to_agg:
+ 2022:
+ other_fossil_fuel_chp: "methane"
+ waste_chp: "biomass"
+ biomass_chp: "biomass"
+ electricity_direct_heating: "electricity_direct_heating"
+ gas_boiler: "methane"
+ methane_chp: "methane"
+ heat_pump: "heat_pump"
+ solar_thermal: "solar_thermal"
+ hydrogen_chp: "hydrogen"
+ hydrogen_boiler: "hydrogen"
+ 2045:
+ waste_chp: "biomass"
+ biomass_chp: "biomass"
+ electricity_direct_heating: "electricity_direct_heating"
+ gas_boiler: "methane"
+ methane_chp: "biomass"
+ heat_pump: "heat_pump"
+ solar_thermal: "solar_thermal"
+ hydrogen_chp: "biomass"
+ hydrogen_boiler: "hydrogen"
diff --git a/digipipe/store/datasets/demand_heat_region/create.smk b/digipipe/store/datasets/demand_heat_region/create.smk
new file mode 100644
index 00000000..915d58b0
--- /dev/null
+++ b/digipipe/store/datasets/demand_heat_region/create.smk
@@ -0,0 +1,695 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import json
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+from pathlib import Path
+from digipipe.scripts.data_io import load_json
+from digipipe.scripts.geo import clip_raster, raster_zonal_stats
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ get_abs_store_root_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ PATH_TO_REGION_DISTRICTS_GPKG
+)
+from digipipe.scripts.datasets.demand import (
+ demand_prognosis,
+ disaggregate_demand_to_municipality,
+ normalize_filter_timeseries
+)
+
+DATASET_PATH = get_abs_dataset_path(
+ "datasets", "demand_heat_region", data_dir=True
+)
+
+rule raster_clip:
+ """
+ Clip raster file to boundary
+ """
+ input:
+ heat_demand=get_abs_dataset_path(
+ "preprocessed", "seenergies_peta5") / "data" / "HD_2015_{sector}_Peta5_0_1_GJ.tif",
+ boundary=get_abs_store_root_path() / "datasets" / "{boundary_dataset}" / "data" / "{boundary_dataset}.gpkg"
+ output:
+ heat_demand = DATASET_PATH / "HD_2015_{sector}_Peta5_0_1_GJ_{boundary_dataset}.tif"
+ run:
+ clip_raster(
+ raster_file_in=input.heat_demand,
+ clip_file=input.boundary,
+ raster_file_out=output.heat_demand,
+ )
+
+rule raster_convert_to_mwh:
+ """
+ Convert TIFF raster to geopackage and change unit from GJ to MWh
+ Note: Unit conversion takes place after clipping as it takes too long to
+ apply to entire dataset for Europe.
+ """
+ input: DATASET_PATH / "HD_2015_{sector}_Peta5_0_1_GJ_{boundary_dataset}.tif"
+ output: DATASET_PATH / "HD_2015_{sector}_Peta5_0_1_MWh_{boundary_dataset}.tif"
+ shell:
+ """
+ gdal_calc.py -A {input} --A_band=1 --outfile={output} --calc="A/3.6"
+ rm {input}
+ """
+
+rule create_raster_zonal_stats:
+ """
+ Create zonal heat statistics (heat demand per geographical unit)
+ """
+ input:
+ heat_demand=DATASET_PATH / "HD_2015_{sector}_Peta5_0_1_MWh_{boundary_dataset}.tif",
+ clip_file=get_abs_store_root_path() / "datasets" / "{boundary_dataset}" / "data" / "{boundary_dataset}.gpkg"
+ output:
+ heat_demand=DATASET_PATH / "demand_heat_zonal_stats-{sector}-{boundary_dataset}.gpkg"
+ run:
+ raster_zonal_stats(
+ raster_file_in=input.heat_demand,
+ clip_file=input.clip_file,
+ zonal_file_out=output.heat_demand,
+ var_name="heat_demand",
+ stats="sum"
+ )
+
+SECTOR_MAPPING = {"hh": "res", "cts": "ser"}
+
+rule heat_demand_shares:
+ """
+ Calculate buildings' heat demand shares of federal state, region and
+ municipalities by merging zonal statistics (for HH and CTS)
+ """
+ input:
+ state=lambda wildcards: DATASET_PATH / f"demand_heat_zonal_stats-{SECTOR_MAPPING[wildcards.sector]}-bkg_vg250_state.gpkg",
+ federal_states=lambda wildcards: DATASET_PATH / f"demand_heat_zonal_stats-{SECTOR_MAPPING[wildcards.sector]}-bkg_vg250_federal_states.gpkg",
+ region_muns=lambda wildcards: DATASET_PATH / f"demand_heat_zonal_stats-{SECTOR_MAPPING[wildcards.sector]}-bkg_vg250_muns_region.gpkg"
+ output:
+ #demand_shares=lambda wildcards: SECTOR_MAPPING[wildcards.sector]
+ demand_shares=DATASET_PATH / "demand_heat_shares_{sector}.json"
+ run:
+ # Read demands
+ demand_state = gpd.read_file(input.state)
+ demand_federal_state = gpd.read_file(input.federal_states)
+ demand_region_muns = gpd.read_file(input.region_muns)
+
+ demand_state = float(demand_state.heat_demand)
+ demand_federal_state = float(
+ demand_federal_state.loc[
+ demand_federal_state.nuts == "DEE"].heat_demand)
+ demand_region_muns = demand_region_muns.heat_demand
+
+ # Calculate shares
+ demand_shares = {
+ "federal_state_of_state": demand_federal_state / demand_state,
+ "region_of_federal_state": (
+ demand_region_muns.sum() / demand_federal_state
+ ),
+ "muns_of_region": (
+ demand_region_muns / demand_region_muns.sum()
+ ).to_dict()
+ }
+
+ # Dump
+ with open(output.demand_shares, "w", encoding="utf8") as f:
+ json.dump(demand_shares, f, indent=4)
+
+rule heat_demand_hh_cts:
+ """
+ Calculate absolute heat demands for municipalities for HH and CTS
+ """
+ input:
+ demand_germany=get_abs_dataset_path(
+ "preprocessed", "ageb_energy_balance") / "data" /
+ "ageb_energy_balance_germany_{sector}_twh_2021.csv",
+ demand_shares=DATASET_PATH / "demand_heat_shares_{sector}.json",
+ demand_future_TN=get_abs_dataset_path(
+ "preprocessed", "bmwk_long_term_scenarios") / "data" /
+ "TN-Strom_buildings_heating_demand_by_carrier_reformatted.csv",
+ demand_future_T45=get_abs_dataset_path(
+ "preprocessed", "bmwk_long_term_scenarios") / "data" /
+ "T45-Strom_buildings_heating_demand_by_carrier_reformatted.csv",
+ output:
+ DATASET_PATH / "demand_{sector}_heat_demand.csv"
+ run:
+ ### Demand 2021 ###
+ demand_germany=pd.read_csv(input.demand_germany, index_col="carrier")
+ with open(input.demand_shares, "r") as f:
+ demand_shares=json.load(f)
+
+ # Calc demand share for each municipality
+ demand_muns = (
+ pd.DataFrame.from_dict(
+ demand_shares.get("muns_of_region"),
+ orient="index"
+ )
+ * demand_shares.get("federal_state_of_state")
+ * demand_shares.get("region_of_federal_state")
+ * demand_germany.drop("Strom", axis=0)[
+ ["space_heating", "hot_water", "process_heat"]
+ ].sum().sum()
+ * 1e6 # TWh to MWh
+ )
+ demand_muns.index.name = "municipality_id"
+ demand_muns.rename(columns={0: 2022}, inplace=True)
+
+ ### Demand 2045: Use reduction factor ###
+ demand_muns = demand_muns.join(
+ demand_prognosis(
+ demand_future_T45=input.demand_future_T45,
+ demand_future_TN=input.demand_future_TN,
+ demand_region=demand_muns,
+ year_base=2022,
+ year_target=2045,
+ scale_by="total",
+ ).rename(columns={2022: 2045})
+ )
+ print(
+ f"Total heat demand for sector {wildcards.sector} in TWh:\n",
+ demand_muns.sum() / 1e6
+ )
+
+ # Dump as CSV
+ demand_muns.to_csv(output[0])
+
+rule heat_demand_ind:
+ """
+ Calculate absolute heat demands for municipalities for Industry
+ """
+ input:
+ demand_heat_ind_germany=get_abs_dataset_path(
+ "preprocessed", "ageb_energy_balance") / "data" /
+ "ageb_energy_balance_germany_ind_twh_2021.csv",
+ demand_ind_states=rules.preprocessed_regiostat_extract_demand_ind.output.demand_states,
+ demand_ind_districts=rules.preprocessed_regiostat_extract_demand_ind.output.demand_districts,
+ lau_codes=rules.preprocessed_eurostat_lau_create.output,
+ employment=get_abs_dataset_path("datasets","employment_region") /
+ "data" / "employment.csv",
+ demand_future_TN=get_abs_dataset_path(
+ "preprocessed","bmwk_long_term_scenarios"
+ ) / "data" / "TN-Strom_ind_demand_reformatted.csv",
+ demand_future_T45=get_abs_dataset_path(
+ "preprocessed","bmwk_long_term_scenarios"
+ ) / "data" / "T45-Strom_ind_demand_reformatted.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG
+ output:
+ DATASET_PATH / "demand_ind_heat_demand.csv"
+ run:
+ # Industrial heat demand Germany
+ demand_heat_ind_germany = pd.read_csv(
+ input.demand_heat_ind_germany, index_col="carrier"
+ )
+ demand_heat_ind_germany = demand_heat_ind_germany.drop("Strom", axis=0)[
+ ["space_heating", "hot_water", "process_heat"]
+ ].sum().sum() * 1e6 # TWh to MWh
+
+ # Industrial energy demand federal states and districts
+ demand_ind_germany = pd.read_csv(
+ input.demand_ind_states, index_col="lau_code", dtype={"lau_code": str}
+ ).total.sum()
+ demand_ind_districts = pd.read_csv(
+ input.demand_ind_districts, index_col="lau_code", dtype={"lau_code": str}
+ )
+
+ # Get region's NUTS codes
+ districts = gpd.read_file(input.region_districts)
+ # Get region's LAU codes
+ lau_codes_region = pd.read_csv(
+ input.lau_codes[0], dtype={"lau_code": str}, usecols=["lau_code", "nuts_code"]
+ )
+ lau_codes_region = lau_codes_region.loc[
+ lau_codes_region.nuts_code.isin(districts.nuts.to_list())]
+ lau_codes_region["lau_code"] = lau_codes_region.lau_code.apply(lambda _: _[:5])
+ lau_codes_region = lau_codes_region.groupby("lau_code").first()
+
+ # Calculate industrial demand using LAU codes of region
+ demand_ind_districts = demand_ind_districts.loc[
+ lau_codes_region.index.to_list()].join(lau_codes_region)
+ demand_ind_districts = demand_ind_districts[["nuts_code", "total"]].set_index("nuts_code")
+ demand_ind_districts = demand_ind_districts.assign(
+ demand_heat_share_germany=demand_ind_districts.total.div(demand_ind_germany)
+ )
+ demand_ind_districts = demand_ind_districts.assign(
+ demand_heat=demand_ind_districts.demand_heat_share_germany.mul(demand_heat_ind_germany)
+ )
+ demand_ind_districts = demand_ind_districts.rename(
+ columns={"demand_heat": "demand_districts"})[["demand_districts"]]
+
+ # Disggregate using employees in industry sector
+ demand_muns = disaggregate_demand_to_municipality(
+ demand_districts=demand_ind_districts,
+ muns=gpd.read_file(input.region_muns),
+ districts=districts,
+ disagg_data=pd.read_csv(input.employment, index_col=0),
+ disagg_data_col="employees_ind"
+ ).rename(columns={"employees_ind": 2022})
+
+ demand_muns.index.name = "municipality_id"
+ demand_muns.rename(columns={0: 2022}, inplace=True)
+
+ ### Demand 2045: Use reduction factor ###
+ demand_muns = demand_muns.join(
+ demand_prognosis(
+ demand_future_T45=input.demand_future_T45,
+ demand_future_TN=input.demand_future_TN,
+ demand_region=demand_muns,
+ year_base=2022,
+ year_target=2045,
+ scale_by="total",
+ ).rename(columns={2022: 2045})
+ )
+ print(
+ f"Total heat demand for sector industry in TWh:\n",
+ demand_muns.sum() / 1e6
+ )
+
+ # Dump as CSV
+ demand_muns.to_csv(output[0])
+
+rule normalize_timeseries:
+ """
+ Extract heat demand timeseries for districts, merge and normalize them
+ """
+ input:
+ timeseries=get_abs_dataset_path("preprocessed", "demandregio") /
+ "data" / "dr_{sector}_gas_timeseries_2011.csv",
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG
+ output:
+ timeseries=DATASET_PATH / "demand_{sector}_heat_timeseries.csv"
+ run:
+ normalize_filter_timeseries(
+ infile=input.timeseries,
+ outfile=output.timeseries,
+ region_nuts=gpd.read_file(input.region_districts).nuts.to_list(),
+ )
+
+rule district_heating:
+ """
+ Calculate heat demands using district heating shares
+ """
+ input:
+ heat_demand=DATASET_PATH / "demand_{sector}_heat_demand.csv"
+ output:
+ heat_demand_cen=DATASET_PATH / "demand_{sector}_heat_demand_cen.csv",
+ heat_demand_dec=DATASET_PATH/ "demand_{sector}_heat_demand_dec.csv"
+ run:
+ print(
+ f"Split heat demand into central and decentral heat demand for "
+ f"sector: {wildcards.sector}"
+ )
+ heat_demand = pd.read_csv(
+ input.heat_demand,
+ index_col="municipality_id",
+ )
+ ds_shares = pd.DataFrame.from_dict(
+ config["district_heating_share"].get(wildcards.sector),
+ orient="index",
+ columns=["district_heating_share"]
+ )
+
+ # Check municipality IDs
+ if not all(mun_id in heat_demand.index for mun_id in ds_shares.index):
+ raise ValueError(
+ "One or more municipality IDs from district_heating_share are "
+ "not found in the heat demand data."
+ )
+
+ # Calculate district heating and decentral heating demand
+ heat_demand_cen = heat_demand.mul(
+ ds_shares.district_heating_share, axis=0)
+ heat_demand_dec = heat_demand.mul(
+ (1 - ds_shares.district_heating_share), axis=0)
+
+ # Dump
+ heat_demand_cen.to_csv(output.heat_demand_cen)
+ heat_demand_dec.to_csv(output.heat_demand_dec)
+
+rule heating_structure_hh_cts:
+ """
+ Create heating structure for households and CTS: demand per technology
+ """
+ input:
+ demand_future_TN=get_abs_dataset_path(
+ "preprocessed", "bmwk_long_term_scenarios") / "data" /
+ "TN-Strom_buildings_heating_demand_by_carrier_reformatted.csv",
+ demand_future_T45=get_abs_dataset_path(
+ "preprocessed","bmwk_long_term_scenarios") / "data" /
+ "T45-Strom_buildings_heating_demand_by_carrier_reformatted.csv",
+ rel_capacities_biomass_dec=get_abs_dataset_path(
+ "preprocessed","dbfz_biomass_capacity_rel") / "data" /
+ "dbfz_biomass_capacity_rel_decentral.csv",
+ generation_cen_TN45=get_abs_dataset_path(
+ "preprocessed","bmwk_long_term_scenarios") / "data" /
+ "T45-Strom_Generation_Heatgrids_Germany_reformatted.csv",
+ rel_capacities_biomass_cen=get_abs_dataset_path(
+ "preprocessed","dbfz_biomass_capacity_rel") / "data" /
+ "dbfz_biomass_capacity_rel_central.csv"
+ output:
+ heating_structure_cen=(
+ #DATASET_PATH / "demand_{sector}_heat_structure_cen.csv"
+ DATASET_PATH / "demand_heat_structure_cen.csv"
+ ),
+ heating_structure_esys_cen=(
+ DATASET_PATH/ "demand_heat_structure_esys_cen.csv"
+ ),
+ heating_structure_dec=(
+ #DATASET_PATH / "demand_{sector}_heat_structure_dec.csv"
+ DATASET_PATH / "demand_heat_structure_dec.csv"
+ ),
+ heating_structure_esys_dec=(
+ DATASET_PATH/ "demand_heat_structure_esys_dec.csv"
+ )
+ run:
+ def get_demand_dec():
+ # Get data from future scenarios
+ demand_future_T45 = pd.read_csv(
+ input.demand_future_T45
+ ) # .set_index(["year", "carrier"])
+ demand_future_TN = pd.read_csv(
+ input.demand_future_TN
+ ) # .set_index(["year", "carrier"])
+
+ # Interpolate for base year
+ demand_future = pd.concat(
+ [
+ demand_future_TN.loc[demand_future_TN.year == 2020],
+ demand_future_T45,
+ ],
+ axis=0,
+ )
+ demand_future = demand_future.set_index(["year", "carrier"]).append(
+ pd.DataFrame(
+ index=pd.MultiIndex.from_product(
+ [[2022], demand_future.carrier.unique(), [np.nan]],
+ names=demand_future.columns,
+ )
+ )
+ )
+ demand_future.sort_index(inplace=True)
+ demand_future = demand_future.unstack(level=1).interpolate().stack()
+
+ # Calculate heating structure
+ demand_future = demand_future.loc[
+ config["heating_structure"].get("years")
+ ].reset_index()
+
+ demand_total = (
+ demand_future[["year", "demand"]].groupby("year").sum()
+ )
+ demand_dec = demand_future.loc[
+ demand_future.carrier != "district_heating"
+ ]
+ # Drop auxiliary power
+ demand_dec = demand_dec.loc[
+ demand_dec.carrier != "electricity_auxiliary"
+ ]
+ demand_total_dec = (
+ demand_dec[["year", "demand"]].groupby("year").sum()
+ )
+ demand_dec = demand_dec.set_index("year").assign(
+ demand_rel=demand_dec.set_index("year").demand.div(
+ demand_total_dec.demand
+ )
+ )
+ demand_dec.drop(columns=["demand"], inplace=True)
+
+ # demand_total_cen = demand_total - demand_total_dec
+
+ return demand_dec
+
+ def get_demand_dec_esys(demand_dec):
+ # Aggregate heat pump demand
+ demand_dec.carrier = demand_dec.carrier.replace(
+ {
+ "ambient_heat_heat_pump": "heat_pump",
+ "electricity_heat_pump": "heat_pump",
+ }
+ )
+ demand_dec = (
+ demand_dec.reset_index().groupby(["year", "carrier"]).sum()
+ )
+
+ # Reset index
+ demand_dec.reset_index(inplace=True)
+
+ # Read relative capacities
+ rel_capacities_biomass_dec = pd.read_csv(
+ input.rel_capacities_biomass_dec, index_col="year"
+ )
+ rel_capacities_biomass_dec.reset_index(inplace=True)
+
+ # Get years from config
+ years = config["heating_structure"].get("years")
+
+ # Get relative demand per conversion technology
+ # In the following, it is assumed for simplification that the
+ # distribution over the capacity corresponds to that of the energy
+ # amount. For this to be the case, the full load hours of all
+ # conversion plants would have to be the same, but in reality they
+ # are not.
+ for year in years:
+ (
+ rel_capacities_biomass_dec,
+ demand_dec,
+ ) = get_rel_demand_conv_tech_biomass(
+ rel_capacities_biomass_dec, demand_dec, year
+ )
+
+ demand_dec_esys = update_rel_demand_with_conv_tech_biomass(
+ rel_capacities_biomass_dec, demand_dec
+ )
+
+ return demand_dec_esys
+
+ def get_demand_cen():
+ # Read central heating generation per technology
+ heat_gen_cen_T45 = pd.read_csv(input.generation_cen_TN45)
+
+ # Read relative capacities
+ rel_capacities_biomass_cen = pd.read_csv(
+ input.rel_capacities_biomass_cen,
+ index_col="year",
+ )
+
+ # Change year 2021 to 2022 neglecting changes in generation
+ # structure during that period
+ heat_gen_cen_T45.loc[
+ heat_gen_cen_T45["year"] == 2021, "year"
+ ] = 2022
+
+ rel_demand_cen = pd.DataFrame()
+ rel_demand_cen_esys = pd.DataFrame()
+
+ for year in config["heating_structure"].get("years"):
+ # Filter data by year
+ heat_gen_cen_T45_year = heat_gen_cen_T45[
+ heat_gen_cen_T45["year"] == year
+ ]
+
+ # Get all carriers used in technology to aggregate
+ carrier_to_agg = config["carrier_to_agg"][year]
+
+ # Get all unique carriers provided in the values of
+ # carrier_to_agg dict
+ unique_carrier = list(set(carrier_to_agg.values()))
+
+ # Rename values in technology col according to aggregation
+ # convention in carrier_to_agg
+ for index, row in heat_gen_cen_T45_year.iterrows():
+ if row["technology"] in carrier_to_agg:
+ heat_gen_cen_T45_year.loc[
+ index, "technology"
+ ] = carrier_to_agg[row["technology"]]
+
+ # Aggregate by carrier / technology according to carrier_to_agg
+ heat_gen_cen_T45_year = heat_gen_cen_T45_year.groupby(
+ "technology"
+ ).agg(
+ {
+ "generation": "sum",
+ "year": "first",
+ }
+ )
+
+ # Reset the index
+ heat_gen_cen_T45_year.reset_index(inplace=True)
+
+ # Rename column 'technology' to 'carrier' and 'generation' to
+ # 'demand_rel'
+ heat_gen_cen_T45_year.rename(
+ {"generation": "demand_rel", "technology": "carrier"},
+ axis=1,
+ inplace=True,
+ )
+
+ # Get relative demand for all carriers
+ rel_demand_cen_year = get_relative_demand(heat_gen_cen_T45_year)
+ rel_demand_cen_year = order_cols_df(rel_demand_cen_year)
+ rel_demand_cen = pd.concat(
+ [rel_demand_cen, rel_demand_cen_year], ignore_index=True
+ )
+
+ # Get all unique carriers listed in raw data
+ carriers = list(heat_gen_cen_T45_year["carrier"].unique())
+
+ # Get unknown carriers if raw data contains carriers that are
+ # not in the energy system
+ unknown_carriers = list(set(carriers) - set(unique_carrier))
+
+ # Drop all unknown carriers
+ for unknown_carrier in unknown_carriers:
+ heat_gen_cen_T45_year.drop(
+ heat_gen_cen_T45_year[
+ heat_gen_cen_T45_year["carrier"] == unknown_carrier
+ ].index,
+ inplace=True,
+ )
+
+ rel_demand_cen_esys_year = get_relative_demand(
+ heat_gen_cen_T45_year
+ )
+ rel_demand_cen_esys_year = order_cols_df(
+ rel_demand_cen_esys_year
+ )
+ rel_demand_cen_esys = pd.concat(
+ [rel_demand_cen_esys, rel_demand_cen_esys_year],
+ ignore_index=True,
+ )
+
+ (
+ rel_capacities_biomass_cen,
+ rel_demand_cen_esys,
+ ) = get_rel_demand_conv_tech_biomass(
+ rel_capacities_biomass_cen, rel_demand_cen_esys, year
+ )
+
+ rel_demand_cen_esys = update_rel_demand_with_conv_tech_biomass(
+ rel_capacities_biomass_cen, rel_demand_cen_esys
+ )
+
+ return rel_demand_cen, rel_demand_cen_esys
+
+ def get_relative_demand(_df):
+ grouped_df = _df.groupby(by=["carrier", "year"]).sum(
+ numeric_only=True
+ )
+
+ # Calculate relative demand from absolute values
+ rel_demand_df = grouped_df.apply(
+ lambda rel_demand_df: rel_demand_df / rel_demand_df.sum()
+ )
+
+ return rel_demand_df
+
+ def order_cols_df(_df):
+ # Drop the index
+ _df.reset_index(inplace=True)
+
+ # Set index on the year to move the column to the front
+ _df.set_index("year", inplace=True)
+ _df.reset_index(inplace=True)
+
+ return _df
+
+ def get_rel_demand_conv_tech_biomass(
+ rel_capacities_biomass, rel_demand, year
+ ):
+ if rel_capacities_biomass.index.name == "year":
+ rel_capacities_biomass.reset_index(inplace=True)
+
+ # Get relative demand of biomass in year
+ rel_demand_biomass = rel_demand.loc[
+ rel_demand["carrier"].eq("biomass")
+ & rel_demand["year"].eq(year),
+ "demand_rel",
+ ]
+
+ # Calculate relative capacities of biomass conversion technologies
+ # with the relative demand of biomass
+ rel_capacities_biomass.loc[
+ rel_capacities_biomass["year"] == year, "capacity_rel"
+ ] *= rel_demand_biomass.values[0]
+
+ return rel_capacities_biomass, rel_demand
+
+ def update_rel_demand_with_conv_tech_biomass(
+ rel_capacities_biomass, rel_demand
+ ):
+ # Rename column "capacity_rel" to "demand_rel"
+ rel_capacities_biomass.rename(
+ {"capacity_rel": "demand_rel"}, axis=1, inplace=True
+ )
+
+ # Merge Dataframe with relative capacities with the one with
+ # relative demands
+ rel_demand_updated = pd.concat(
+ [rel_demand, rel_capacities_biomass], ignore_index=True
+ )
+
+ # Drop redundant entry for biomass
+ rel_demand_updated.drop(
+ rel_demand_updated[
+ rel_demand_updated["carrier"] == "biomass"
+ ].index,
+ inplace=True,
+ )
+
+ # Sort by year
+ rel_demand_updated.sort_values(
+ by=["year"], inplace=True, ignore_index=True
+ )
+
+ return rel_demand_updated
+
+ # Get relative demand of decentral heat
+ demand_dec = get_demand_dec()
+
+ # Dump heating structure (for info)
+ demand_dec.to_csv(output.heating_structure_dec)
+
+ # Get relative demand of decentral heat of esys
+ demand_dec_esys = get_demand_dec_esys(demand_dec)
+
+ # Dump heating structure (for esys)
+ demand_dec_esys.to_csv(output.heating_structure_esys_dec, index=False)
+
+ # Get relative demand of central heat
+ rel_demand_cen, rel_demand_cen_esys = get_demand_cen()
+
+ # Dump heating structure (for info)
+ rel_demand_cen.to_csv(
+ output.heating_structure_cen,
+ index=False,
+ )
+
+ # Dump heating structure (for esys)
+ rel_demand_cen_esys.to_csv(
+ output.heating_structure_esys_cen,
+ index=False,
+ )
+
+rule create_captions:
+ """
+ Create attribute captions for app
+ """
+ input:
+ demand=rules.datasets_demand_heat_region_heating_structure_hh_cts.output,
+ bmwk_lts=rules.preprocessed_bmwk_long_term_scenarios_create_captions.output[0]
+ output: DATASET_PATH / "demand_heat_region_attribute_captions.json"
+ run:
+ bmwk_lts = load_json(input.bmwk_lts)
+ captions = {
+ "datasets_caption_map": {
+ Path(f).stem: "demand_heat" for f in input.demand
+ },
+ "captions": {
+ "demand_heat":
+ bmwk_lts["captions"]["bmwk_long_term_scenarios"]
+ }
+ }
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(captions, f, indent=4)
diff --git a/digipipe/store/datasets/demand_heat_region/data/.gitkeep b/digipipe/store/datasets/demand_heat_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/demand_heat_region/dataset.md b/digipipe/store/datasets/demand_heat_region/dataset.md
new file mode 100644
index 00000000..694607e2
--- /dev/null
+++ b/digipipe/store/datasets/demand_heat_region/dataset.md
@@ -0,0 +1,148 @@
+# Wärmebedarf
+
+Wärmebedarfe (Endenergie) Fernwärme und dezentrale Wärme sowie Wärmezeitreihen
+für Haushalte, GHD und Industrie je Gemeinde.
+
+## Gesamtwärmebedarf
+
+Die Berechnung der regionalen Prognosewerte je Verbrauchssektor erfolgt anhand
+landesweiter Prognosen aus den
+[BMWK Langfristszenarien](../../preprocessed/bmwk_long_term_scenarios/dataset.md).
+
+### Haushalte
+
+- Jährlicher Wärmebedarf je Gemeinde in MWh: Bundeswert aus
+ [AG Energiebilanzen](../../preprocessed/ageb_energy_balance/dataset.md)
+ 2021 für Raumwärme, Warmwasser und Prozesswärme, desaggregiert auf Gemeinden
+ mittels Wärmebedarfs-Rasterdaten aus 2015 (Wärmebedarfsdichte 1ha) aus
+ [Peta5](../../raw/seenergies_peta5/dataset.md).
+ Anm.: Die Desaggregation könnte alternativ über Zensus "Gebäude mit Wohnraum
+ nach Heizungsart" (31231-02-01-5, s.
+ [regiostat](../../raw/regiostat/dataset.md) erfolgen)
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion der
+ Gebäudewärmebedarfe aus
+ [BMWK Langfristszenarien](../../preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Hierbei wird das Szenario "TN-Strom" als Grundlage für den Status
+ quo verwendet und Werte für 2022 interpoliert. Die Zielwerte werden dem
+ Szenario "T45-Strom" entnommen.
+- Gemittelte, normierte Gasbedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../preprocessed/demandregio/dataset.md)-Daten von 2022 die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird, da die Basis
+ SLP-Profile sind und Differenzen zwischen verschiedenen Jahren nur aufgrund
+ der Lage von Wochenenden und Feiertagen bestehen. Diese werden daher
+ vernachlässigt.
+
+### GHD
+
+- Jährlicher Wärmebedarf je Gemeinde in MWh: Bundeswert aus
+ [AG Energiebilanzen](../../preprocessed/ageb_energy_balance/dataset.md)
+ 2021 für Raumwärme, Warmwasser und Prozesswärme, desaggregiert auf Gemeinden
+ mittels Wärmebedarfs-Rasterdaten aus 2015 (Wärmebedarfsdichte 1ha) aus
+ [Peta5](../../raw/seenergies_peta5/dataset.md)
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion der
+ Gebäudewärmebedarfe aus
+ [BMWK Langfristszenarien](../../preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Hierbei wird das Szenario "TN-Strom" als Grundlage für den Status
+ quo verwendet und Werte für 2022 interpoliert. Die Zielwerte werden dem
+ Szenario "T45-Strom" entnommen.
+- Gemittelte, normierte Gasbedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../preprocessed/demandregio/dataset.md)-Daten von 2022 die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird, da die Basis
+ SLP-Profile sind und Differenzen zwischen verschiedenen Jahren nur aufgrund
+ der Lage von Wochenenden und Feiertagen bestehen. Diese werden daher
+ vernachlässigt.
+
+### Industrie
+
+- Jährlicher Wärmebedarf je Gemeinde in MWh: Bundeswert aus
+ [AG Energiebilanzen](../../preprocessed/ageb_energy_balance/dataset.md)
+ 2021 für Raumwärme, Warmwasser und Prozesswärme. Die Desaggregation auf
+ Landkreisebene erfolgt anhand des Gesamtenergiebedarfs im verarbeitenden
+ Gewerbe aus [Regionalstatistik](../../preprocessed/regiostat/dataset.md).
+ Die anschließende Desaggregation auf Gemeindeebene wird mittels
+ Beschäftigtenzahlen im verarbeitenden Gewerbe in 2022 aus
+ [Regionalstatistik](../../preprocessed/regiostat/dataset.md) vorgenommen.
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion des
+ industriellen Gesamtenergiebedarfs aus
+ [BMWK Langfristszenarien](../../preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Im Unterschied zu Haushalten und GHD liegen die Daten für den
+ Wärme- und Stromanteil nicht getrennt vor, sodass auf den
+ Gesamtenergiebedarf zurückgegriffen wird.
+ Es wird das Szenario "TN-Strom" als Grundlage für den Status quo verwendet und
+ Werte für 2022 interpoliert. Die Zielwerte werden dem Szenario "T45-Strom"
+ entnommen.
+- Gemittelte, normierte Gasbedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../preprocessed/demandregio/dataset.md)-Daten von 2022 die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird, da die Basis
+ SLP-Profile sind und Differenzen zwischen verschiedenen Jahren nur aufgrund
+ der Lage von Wochenenden und Feiertagen bestehen. Diese werden daher
+ vernachlässigt.
+- Es erfolgt keine Aufteilung des Wärmebedarfs auf unterschiedliche
+ Temperaturniveaus.
+
+## Dezentrale Wärme und Fernwärme
+
+Der Gesamtwärmebedarf wird auf dezentrale Heizsysteme und Fernwärme aufgeteilt.
+Fernwärmenetze existieren in Dessau-Roßlau, Bitterfeld-Wolfen, Köthen und
+Wittenberg.
+
+Da keine Daten zum tatsächlichen Fernwärmebedarf vorliegen, werden Annahmen auf
+Basis folgender Quellen getroffen:
+
+- [Zensus 2011: Gebäude nach Heizungsart](https://www.regionalstatistik.de/genesis//online?operation=table&code=31211-04-01-5-B)
+- [BMWK Langfristszenarien: Wärmenachfrage in Wärmenetzen (HH&GHD) (2025)](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/54022/62a2667df6f8c176ff129f7ede944837)
+- [STALA ST: Wärmebilanz der Industriebetriebe (2021)](https://statistik.sachsen-anhalt.de/themen/wirtschaftsbereiche/energie-und-wasserversorgung/tabellen-energieverwendung#c256237)
+- [STALA ST: Energie- und Wasserversorgung](https://statistik.sachsen-anhalt.de/fileadmin/Bibliothek/Landesaemter/StaLa/startseite/Themen/Energie/Berichte/6E403_2020-A.pdf)
+- [WindNODE](https://windnode-abw.readthedocs.io/en/latest/energy_system_model.html#district-heating)
+- [Peta5: D5 1 District Heating Areas (2020)](https://s-eenergies-open-data-euf.hub.arcgis.com/datasets/b62b8ad79f0e4ae38f032ad6aadb91a0_0/)
+
+Annahmen zu Fernwärmeanteilen (Anteil der Endenergie aus Fernwärme an gesamter
+Wärme-Endenergie) je Bedarfssektor:
+
+| Fernwärmenetz | Haushalte | GHD | Industrie |
+|-------------------|----------:|-----:|----------:|
+| Dessau-Roßlau | 0,36 | 0,36 | 0,19 |
+| Bitterfeld-Wolfen | 0,11 | 0,11 | 0,21 |
+| Köthen | 0,07 | 0,07 | 0,21 |
+| Wittenberg | 0,15 | 0,15 | 0,01 |
+
+Die Fernwärmeanteile können in der [config.yml](config.yml) im Abschnitt
+`district_heating_share` für jeden Sektor separat angepasst werden. Es wird
+vereinfachend angenommen, dass der Anteil an Fernwärme für alle
+Szenarien/Zieljahre gleich bleibt.
+
+## Beheizungsstruktur
+
+Die Beheizungsstruktur für 2020 und 2045 wird den
+[BMWK Langfristszenarien](../../preprocessed/bmwk_long_term_scenarios/dataset.md)
+entnommen (Gebäude: Haushalte und GHD Energiebedarf) und für 2022 interpoliert.
+Hierbei wird nach Technologien für dezentrale sowie Fernwärme unterschieden.
+Für die Biomasse wird der relative Energiebedarf mit Hilfe von Anteilen der
+installierten Leistung von spezifischen Biomasse-Konversionsanlagen
+[dbfz_biomasss_capacity_rel](../../preprocessed/dbfz_biomass_capacity_rel/dataset.md)
+je Technologie aufgelöst. Der Vereinfachung halber wird angenommen, dass die
+relative installierte Leistung der relativen Energiemenge entspricht.
+
+## Ergebnisdaten
+
+- Haushalte: Wärmebedarf gesamt: `demand_hh_heat_demand.csv`
+- Haushalte: Wärmebedarf Fernwärme: `demand_hh_heat_demand_cen.csv`
+- Haushalte: Wärmebedarf dezentrale Wärme: `demand_hh_heat_demand_dec.csv`
+- Haushalte: Zeitreihen: `demand_hh_heat_timeseries.csv`
+
+- GHD: Wärmebedarf gesamt: `demand_cts_heat_demand.csv`
+- GHD: Wärmebedarf Fernwärme: `demand_cts_heat_demand_cen.csv`
+- GHD: Wärmebedarf dezentrale Wärme: `demand_cts_heat_demand_dec.csv`
+- GHD: Zeitreihen: `demand_cts_heat_timeseries.csv`
+
+- Industrie: Wärmebedarf gesamt: `demand_ind_heat_demand.csv`
+- Industrie: Wärmebedarf Fernwärme: `demand_ind_heat_demand_cen.csv`
+- Industrie: Wärmebedarf dezentrale Wärme: `demand_ind_heat_demand_dec.csv`
+- GHD: Zeitreihen: `demand_ind_heat_timeseries.csv`
+
+- Beheizungsstruktur dezentral (informativ): `demand_heat_structure_dec.csv`
+- Beheizungsstruktur zentral (informativ): `demand_heat_structure_cen.csv`
+- Beheizungsstruktur dezentral für Weiterverwendung im Energiesystem:
+ `demand_heat_structure_esys_dec.csv`
+- Beheizungsstruktur Fernwärme für Weiterverwendung im Energiesystem:
+ `demand_heat_structure_esys_cen.csv`
diff --git a/digipipe/store/datasets/demand_heat_region/scripts/create.py b/digipipe/store/datasets/demand_heat_region/scripts/create.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/emissions_region/config.yml b/digipipe/store/datasets/emissions_region/config.yml
new file mode 100644
index 00000000..17d3af5a
--- /dev/null
+++ b/digipipe/store/datasets/emissions_region/config.yml
@@ -0,0 +1,3 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
diff --git a/digipipe/store/datasets/emissions_region/create.smk b/digipipe/store/datasets/emissions_region/create.smk
new file mode 100644
index 00000000..00666d45
--- /dev/null
+++ b/digipipe/store/datasets/emissions_region/create.smk
@@ -0,0 +1,80 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import json
+import pandas as pd
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path(
+ "datasets", "emissions_region", data_dir=True)
+
+rule create_chart_data:
+ """
+ Extract and aggregate emissions for charts
+ """
+ input:
+ get_abs_dataset_path("raw", "emissions") / "data" / "emissions.csv"
+ output:
+ DATASET_PATH / "emissions_chart_overview.json"
+ run:
+ emissions = pd.read_csv(
+ input[0],
+ header=[0, 1],
+ index_col=[0, 1, 2, 3],
+ )
+
+ emissions_dict = dict(
+ # Sector: Energy industry (CRF 1.A.1 + 1.B)
+ **emissions.loc[[
+ (1, "A", "1", "energy_industry"),
+ (1, "B", "1", "diffuse_emissions_solid_fuels"),
+ (1, "B", "2", "diffuse_emissions_fuel_oil_natural_gas"),
+ ]].sum().to_frame(
+ name="energy_industry").round(1).to_dict(orient="list"),
+
+ # Sector: Industry (CRF 1.A.2 + 2)
+ **emissions.loc[[
+ (1, "A", "2", "industry"),
+ (2, "*", "*", "process_emissions"),
+ ]].sum().to_frame(
+ name="industry").round(1).to_dict(orient="list"),
+
+ # Sector: Traffic (CRF 1.A.3)
+ **emissions.loc[
+ (1, "A", "3", "traffic"),
+ ].to_frame(
+ name="traffic").round(1).to_dict(orient="list"),
+
+ # Sector: Buildings (CRF 1.A.4 + 1.A.5)
+ **emissions.loc[
+ (1, "A", "4-5", "buildings_firing"),
+ ].to_frame(
+ name="buildings_firing").round(1).to_dict(orient="list"),
+
+ # Sector: Agriculture (CRF 3)
+ **emissions.groupby(
+ "sector"
+ ).sum().loc[3].to_frame(
+ name="agricultural").round(1).to_dict(orient="list"),
+
+ # Sector: Waste and waste water (CRF 5)
+ **emissions.loc[
+ (5, "*", "*", "waste_waste_water"),
+ ].to_frame(
+ name="waste_waste_water").round(1).to_dict(orient="list"),
+ )
+
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(emissions_dict, f, indent=4)
+
+rule copy_emissions:
+ """
+ Copy raw emissions file
+ """
+ input:
+ get_abs_dataset_path("raw", "emissions") / "data" / "emissions.csv"
+ output:
+ DATASET_PATH / "emissions.csv"
+ shell: "cp -p {input} {output}"
diff --git a/digipipe/store/datasets/emissions_region/data/.gitkeep b/digipipe/store/datasets/emissions_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/emissions_region/dataset.md b/digipipe/store/datasets/emissions_region/dataset.md
new file mode 100644
index 00000000..952e1bc4
--- /dev/null
+++ b/digipipe/store/datasets/emissions_region/dataset.md
@@ -0,0 +1,8 @@
+# Emissionen
+
+Emissionen für Sachsen-Anhalt und die Region, aggregiert nach Sektoren der
+CRF-Nomenklatur.
+
+Datei `emissions.json` enthält Chartdaten.
+
+Raw dataset: [emissions](../../raw/emissions/dataset.md)
diff --git a/digipipe/store/datasets/employment_region/config.yml b/digipipe/store/datasets/employment_region/config.yml
new file mode 100644
index 00000000..17d3af5a
--- /dev/null
+++ b/digipipe/store/datasets/employment_region/config.yml
@@ -0,0 +1,3 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
diff --git a/digipipe/store/datasets/employment_region/create.smk b/digipipe/store/datasets/employment_region/create.smk
new file mode 100644
index 00000000..f585d5e6
--- /dev/null
+++ b/digipipe/store/datasets/employment_region/create.smk
@@ -0,0 +1,25 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG
+)
+
+DATASET_PATH = get_abs_dataset_path("datasets", "employment_region")
+
+rule create:
+ """
+ Create employment dataset for region
+ """
+ input:
+ employment_total=get_abs_dataset_path("preprocessed", "ba_employment") / "data" / "employment_muns.csv",
+ employment_ind=get_abs_dataset_path("preprocessed", "regiostat") / "data" / "employment_industry_muns.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG
+ output:
+ DATASET_PATH / "data" / "employment.csv"
+ script:
+ DATASET_PATH / "scripts" / "create.py"
diff --git a/digipipe/store/datasets/employment_region/data/.gitkeep b/digipipe/store/datasets/employment_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/employment_region/dataset.md b/digipipe/store/datasets/employment_region/dataset.md
new file mode 100644
index 00000000..8b913f2e
--- /dev/null
+++ b/digipipe/store/datasets/employment_region/dataset.md
@@ -0,0 +1,8 @@
+# Sozialversicherungspflichtig Beschäftigte und Betriebe
+
+Gesamtanzahl sozialversicherungspflichtig Beschäftigte und Betriebsstätten
+je Gemeinde für die Region.
+
+Raw datasets:
+[ba_employment](../../raw/ba_employment/dataset.md),
+[regiostat](../../raw/regiostat/dataset.md)
diff --git a/digipipe/store/datasets/employment_region/scripts/create.py b/digipipe/store/datasets/employment_region/scripts/create.py
new file mode 100644
index 00000000..268e0f07
--- /dev/null
+++ b/digipipe/store/datasets/employment_region/scripts/create.py
@@ -0,0 +1,54 @@
+import geopandas as gpd
+import pandas as pd
+
+
+def process() -> None:
+ # Get muns
+ muns = gpd.read_file(snakemake.input.region_muns)
+
+ # Get total employment data
+ employment_total = pd.read_csv(
+ snakemake.input.employment_total, index_col=0, dtype={"ags": str}
+ ).loc[muns.ags.to_list()]
+
+ # Get industrial employment data
+ employment_ind = pd.read_csv(
+ snakemake.input.employment_ind, index_col=0, dtype={"ags": str}
+ ).loc[muns.ags.to_list()][["employees_ind", "companies_ind"]]
+
+ # Fill missing data by average employees per company
+ employees_per_company = employment_ind.employees_ind.div(
+ employment_ind.companies_ind
+ )
+ if employees_per_company.isna().sum() > 0:
+ print(
+ f"WARNING: Number of employees or companies missing in "
+ f"{employees_per_company.isna().sum()} of {len(muns)} "
+ f"municipalities! Using average for employees per company "
+ f"({employees_per_company.mean().round(1)}) for those..."
+ )
+ employment_ind = employment_ind.assign(
+ employees_ind=employment_ind.employees_ind.fillna(
+ employment_ind.companies_ind * employees_per_company.mean()
+ )
+ )
+ employment_ind = employment_ind.assign(
+ employees_ind=employment_ind.employees_ind.round().astype(int)
+ )
+
+ # Join total data with industry data
+ employment = employment_total.join(employment_ind)
+
+ # Join employment data with municipality ids
+ employment = (
+ muns.rename(columns={"id": "municipality_id"})
+ .set_index("ags")
+ .merge(employment, left_index=True, right_index=True)
+ .set_index("municipality_id")
+ )[["employees_total", "companies_total", "employees_ind", "companies_ind"]]
+
+ employment.to_csv(snakemake.output[0])
+
+
+if __name__ == "__main__":
+ process()
diff --git a/digipipe/store/datasets/esys_raw/data/.gitkeep b/digipipe/store/datasets/esys_raw/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/heatpump_cop/config.yml b/digipipe/store/datasets/heatpump_cop/config.yml
new file mode 100644
index 00000000..0ab63384
--- /dev/null
+++ b/digipipe/store/datasets/heatpump_cop/config.yml
@@ -0,0 +1,19 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Heatpump parameters
+heatpumps:
+ technology:
+ # Market shares
+ share_ASHP: 0.8
+ share_GSHP: 0.2
+ params:
+ heating_temp: 40
+ quality_grade_ASHP: 0.4
+ quality_grade_GSHP: 0.4
+ icing_factor: 0.8
+ icing_temp: 2
+ # Seasonal performance factors (efficiency gain until 2045)
+ spf_ASHP: 1.25
+ spf_GSHP: 1.25
diff --git a/digipipe/store/datasets/heatpump_cop/create.smk b/digipipe/store/datasets/heatpump_cop/create.smk
new file mode 100644
index 00000000..d32691cb
--- /dev/null
+++ b/digipipe/store/datasets/heatpump_cop/create.smk
@@ -0,0 +1,43 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import pandas as pd
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("datasets", "heatpump_cop")
+
+rule create:
+ """
+ Calculate COPs for air- and ground source heatpumps
+ """
+ input:
+ temperature=rules.preprocessed_dwd_temperature_create.output
+ output:
+ cop_ashp=DATASET_PATH / "data" / "heatpump_cop_ashp_timeseries.csv",
+ cop_gshp=DATASET_PATH / "data" / "heatpump_cop_gshp_timeseries.csv"
+ script:
+ DATASET_PATH / "scripts" / "create.py"
+
+rule merge:
+ """
+ Create one heatpump COP timeseries by weighting both COP timeseries
+ """
+ input: rules.datasets_heatpump_cop_create.output
+ output: DATASET_PATH / "data" / "heatpump_cop_timeseries.csv"
+ run:
+ tech = config["heatpumps"].get("technology")
+ cop = pd.concat(
+ [
+ pd.read_csv(f, index_col=0)
+ for f in input
+ ],
+ axis=1,
+ )
+ cop = pd.Series(
+ cop.cop_ashp * tech.get("share_ASHP") +
+ cop.cop_gshp * tech.get("share_GSHP"),
+ name="cop"
+ ).round(3)
+ cop.to_csv(output[0])
diff --git a/digipipe/store/datasets/heatpump_cop/data/.gitkeep b/digipipe/store/datasets/heatpump_cop/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/heatpump_cop/dataset.md b/digipipe/store/datasets/heatpump_cop/dataset.md
new file mode 100644
index 00000000..92edf018
--- /dev/null
+++ b/digipipe/store/datasets/heatpump_cop/dataset.md
@@ -0,0 +1,35 @@
+# Wärmepumpen COP
+
+Zeitreihe für die Leistungszahl / Coefficient of performance (COP) für
+Wärmepumpen. Berücksichtigt werden Luftwärmepumpen (ASHP) und Erdwärmepumpen
+(GSHP). Der COP wird mit Hilfe von Zeitreihen der Umgebungstemperatur (ASHP)
+bzw. der Bodentemperatur (GSHP) für jeden Zeitschritt berechnet.
+
+Details zur Berechnungsmethodik können der Dokumentation von
+[oemof.thermal](https://oemof-thermal.readthedocs.io/en/latest/compression_heat_pumps_and_chillers.html)
+entnommen werden.
+
+Annahmen:
+
+- Vorlauftemperatur: 40 °C
+- Gütegrad / Quality grade: 0.4 (nach
+ [VDE](https://www.energiedialog2050.info/wp-content/uploads/simple-file-list/VDE_ST_ETG_Warmemarkt_RZ-web.pdf))
+- Vereisungsverluste bei ASHP: 20 % bei <2 °C
+
+Daraus ergibt sich eine mittlere Jahresarbeitszahl (JAZ) von 3,3 für ASHP und
+4,3 für GSHP, die mit typischen Werten für 2019
+([AEW](https://static.agora-energiewende.de/fileadmin/Projekte/2022/2022-04_DE_Scaling_up_heat_pumps/A-EW_273_Waermepumpen_WEB.pdf))
+übereinstimmen. Für das Zukunftsszenario wird ferner ein Effizienzgewinn durch
+technische Weiterentwicklung von 25 % angenommen
+[ewi](https://www.ewi.uni-koeln.de/cms/wp-content/uploads/2015/12/2014_06_24_ENDBER_P7570_Energiereferenzprognose-GESAMT-FIN-IA.pdf).
+
+Beide separat erstelle Zeitreihen werden anhand der heutigen Marktdurchdringung
+gewichtet und in eine mittlere Zeitreihe für Wärmepumpen überführt. Im Jahr
+2022 wurden 87 % ASHP und 13 % GSHP abgesetzt nach
+[BWP](https://www.waermepumpe.de/fileadmin/user_upload/waermepumpe/05_Presse/01_Pressemitteilungen/BWP_Branchenstudie_2023_DRUCK.pdf),
+über die letzten 10 Jahre beträgt das Verhältnis ca. 80:20.
+Für 2045 wird daher ein Anteil von 80 % ASHP und 20 % GSHP angenommen.
+
+Verwendet Datensätze:
+
+- [dwd_temperature](../../preprocessed/dwd_temperature/dataset.md)
diff --git a/digipipe/store/datasets/heatpump_cop/scripts/create.py b/digipipe/store/datasets/heatpump_cop/scripts/create.py
new file mode 100644
index 00000000..ac316dd7
--- /dev/null
+++ b/digipipe/store/datasets/heatpump_cop/scripts/create.py
@@ -0,0 +1,93 @@
+import pandas as pd
+
+
+def calc_heat_pump_cops(
+ t_high: list,
+ t_low: list,
+ quality_grade: float,
+ consider_icing: bool = False,
+ temp_icing: float = None,
+ factor_icing: float = None,
+ spf: float = None,
+) -> list:
+ """Calculate temperature-dependent COP of heat pumps including efficiency
+ gain over time.
+
+ COP-Code was adapted from oemof-thermal:
+ https://github.com/oemof/oemof-thermal/blob/features/cmpr_heatpumps_and_chillers/src/oemof/thermal/compression_heatpumps_and_chillers.py
+
+ Efficiency corrections are based upon increase of seasonal performance
+ factor (SPF) for scenario year since today SQ.
+ """
+
+ # Expand length of lists with temperatures and convert unit to Kelvin
+ length = max([len(t_high), len(t_low)])
+ if len(t_high) == 1:
+ list_t_high_k = [t_high[0] + 273.15] * length
+ elif len(t_high) == length:
+ list_t_high_k = [t + 273.15 for t in t_high]
+ if len(t_low) == 1:
+ list_t_low_k = [t_low[0] + 273.15] * length
+ elif len(t_low) == length:
+ list_t_low_k = [t + 273.15 for t in t_low]
+
+ # Calculate COPs
+ if not consider_icing:
+ cops = [
+ quality_grade * t_h / (t_h - t_l)
+ for t_h, t_l in zip(list_t_high_k, list_t_low_k)
+ ]
+
+ # Temperatures below 2 degC lead to icing at evaporator in
+ # heat pumps working with ambient air as heat source.
+ elif consider_icing:
+ cops = []
+ for t_h, t_l in zip(list_t_high_k, list_t_low_k):
+ if t_l < temp_icing + 273.15:
+ cops = cops + [factor_icing * quality_grade * t_h / (t_h - t_l)]
+ if t_l >= temp_icing + 273.15:
+ cops = cops + [quality_grade * t_h / (t_h - t_l)]
+
+ # Efficiency gain for scenario year
+ if spf is not None:
+ cops = [_ * spf for _ in cops]
+
+ return cops
+
+
+def process() -> None:
+ # Get heatpump params and temperature timeseries
+ hp_params = snakemake.config["heatpumps"].get("params")
+ temp = pd.read_csv(
+ snakemake.input.temperature[0],
+ index_col=0,
+ )
+
+ # Calculate COPs
+ cops_ashp = calc_heat_pump_cops(
+ t_high=[hp_params.get("heating_temp")],
+ t_low=temp.temp_amb.to_list(),
+ quality_grade=hp_params.get("quality_grade_ASHP"),
+ consider_icing=True,
+ temp_icing=hp_params.get("icing_temp"),
+ factor_icing=hp_params.get("icing_factor"),
+ spf=hp_params.get("spf_ASHP"),
+ )
+ cops_gshp = calc_heat_pump_cops(
+ t_high=[hp_params.get("heating_temp")],
+ t_low=temp.temp_soil.to_list(),
+ quality_grade=hp_params.get("quality_grade_GSHP"),
+ spf=hp_params.get("spf_GSHP"),
+ )
+
+ # Round and dump
+ pd.Series(cops_ashp, name="cop_ashp",).round(
+ 3
+ ).to_csv(snakemake.output.cop_ashp)
+ pd.Series(cops_gshp, name="cop_gshp",).round(
+ 3
+ ).to_csv(snakemake.output.cop_gshp)
+
+
+if __name__ == "__main__":
+ process()
diff --git a/digipipe/store/datasets/module.smk b/digipipe/store/datasets/module.smk
new file mode 100644
index 00000000..e294931a
--- /dev/null
+++ b/digipipe/store/datasets/module.smk
@@ -0,0 +1,157 @@
+"""
+Dataset registry for datasets module which is loaded by main snakemake file.
+All datasets in the datasets category must be added to this file.
+
+Template:
+---------
+module :
+ snakefile: "/create.smk"
+ config: config["store"]["datasets"][""]
+use rule * from as datasets__*
+
+"""
+
+module bkg_vg250_districts_region:
+ snakefile: "bkg_vg250_districts_region/create.smk"
+ config: config["store"]["datasets"]["bkg_vg250_districts_region"]
+use rule * from bkg_vg250_districts_region as datasets_bkg_vg250_districts_region_*
+
+module bkg_vg250_muns_region:
+ snakefile: "bkg_vg250_muns_region/create.smk"
+ config: config["store"]["datasets"]["bkg_vg250_muns_region"]
+use rule * from bkg_vg250_muns_region as datasets_bkg_vg250_muns_region_*
+
+module bkg_vg250_state:
+ snakefile: "bkg_vg250_state/create.smk"
+ config: config["store"]["datasets"]["bkg_vg250_state"]
+use rule * from bkg_vg250_state as datasets_bkg_vg250_state_*
+
+module bkg_vg250_federal_states:
+ snakefile: "bkg_vg250_federal_states/create.smk"
+ config: config["store"]["datasets"]["bkg_vg250_federal_states"]
+use rule * from bkg_vg250_federal_states as datasets_bkg_vg250_federal_states_*
+
+module bkg_vg250_region:
+ snakefile: "bkg_vg250_region/create.smk"
+ config: config["store"]["datasets"]["bkg_vg250_region"]
+use rule * from bkg_vg250_region as datasets_bkg_vg250_region_*
+
+module bnetza_mastr_wind_region:
+ snakefile: "bnetza_mastr_wind_region/create.smk"
+ config: config["store"]["datasets"]["bnetza_mastr_wind_region"]
+use rule * from bnetza_mastr_wind_region as datasets_bnetza_mastr_wind_region_*
+
+module bnetza_mastr_pv_ground_region:
+ snakefile: "bnetza_mastr_pv_ground_region/create.smk"
+ config: config["store"]["datasets"]["bnetza_mastr_pv_ground_region"]
+use rule * from bnetza_mastr_pv_ground_region as datasets_bnetza_mastr_pv_ground_region_*
+
+module bnetza_mastr_pv_roof_region:
+ snakefile: "bnetza_mastr_pv_roof_region/create.smk"
+ config: config["store"]["datasets"]["bnetza_mastr_pv_roof_region"]
+use rule * from bnetza_mastr_pv_roof_region as datasets_bnetza_mastr_pv_roof_region_*
+
+module bnetza_mastr_biomass_region:
+ snakefile: "bnetza_mastr_biomass_region/create.smk"
+ config: config["store"]["datasets"]["bnetza_mastr_biomass_region"]
+use rule * from bnetza_mastr_biomass_region as datasets_bnetza_mastr_biomass_region_*
+
+module bnetza_mastr_hydro_region:
+ snakefile: "bnetza_mastr_hydro_region/create.smk"
+ config: config["store"]["datasets"]["bnetza_mastr_hydro_region"]
+use rule * from bnetza_mastr_hydro_region as datasets_bnetza_mastr_hydro_region_*
+
+module bnetza_mastr_combustion_region:
+ snakefile: "bnetza_mastr_combustion_region/create.smk"
+ config: config["store"]["datasets"]["bnetza_mastr_combustion_region"]
+use rule * from bnetza_mastr_combustion_region as datasets_bnetza_mastr_combustion_region_*
+
+module bnetza_mastr_gsgk_region:
+ snakefile: "bnetza_mastr_gsgk_region/create.smk"
+ config: config["store"]["datasets"]["bnetza_mastr_gsgk_region"]
+use rule * from bnetza_mastr_gsgk_region as datasets_bnetza_mastr_gsgk_region_*
+
+module bnetza_mastr_storage_region:
+ snakefile: "bnetza_mastr_storage_region/create.smk"
+ config: config["store"]["datasets"]["bnetza_mastr_storage_region"]
+use rule * from bnetza_mastr_storage_region as datasets_bnetza_mastr_storage_region_*
+
+module bnetza_mastr_captions:
+ snakefile: "bnetza_mastr_captions/create.smk"
+ config: config["store"]["datasets"]["bnetza_mastr_captions"]
+use rule * from bnetza_mastr_captions as datasets_bnetza_mastr_captions_*
+
+module population_region:
+ snakefile: "population_region/create.smk"
+ config: config["store"]["datasets"]["population_region"]
+use rule * from population_region as datasets_population_region_*
+
+module employment_region:
+ snakefile: "employment_region/create.smk"
+ config: config["store"]["datasets"]["employment_region"]
+use rule * from employment_region as datasets_employment_region_*
+
+module demand_electricity_region:
+ snakefile: "demand_electricity_region/create.smk"
+ config: config["store"]["datasets"]["demand_electricity_region"]
+use rule * from demand_electricity_region as datasets_demand_electricity_region_*
+
+module heatpump_cop:
+ snakefile: "heatpump_cop/create.smk"
+ config: config["store"]["datasets"]["heatpump_cop"]
+use rule * from heatpump_cop as datasets_heatpump_cop_*
+
+module demand_heat_region:
+ snakefile: "demand_heat_region/create.smk"
+ config: config["store"]["datasets"]["demand_heat_region"]
+use rule * from demand_heat_region as datasets_demand_heat_region_*
+
+module renewable_feedin:
+ snakefile: "renewable_feedin/create.smk"
+ config: config["store"]["datasets"]["renewable_feedin"]
+use rule * from renewable_feedin as datasets_renewable_feedin_*
+
+module potentialarea_wind_region:
+ snakefile: "potentialarea_wind_region/create.smk"
+ config: config["store"]["datasets"]["potentialarea_wind_region"]
+use rule * from potentialarea_wind_region as datasets_potentialarea_wind_region_*
+
+module potentialarea_pv_ground_region:
+ snakefile: "potentialarea_pv_ground_region/create.smk"
+ config: config["store"]["datasets"]["potentialarea_pv_ground_region"]
+use rule * from potentialarea_pv_ground_region as datasets_potentialarea_pv_ground_region_*
+
+module potentialarea_pv_roof_region:
+ snakefile: "potentialarea_pv_roof_region/create.smk"
+ config: config["store"]["datasets"]["potentialarea_pv_roof_region"]
+use rule * from potentialarea_pv_roof_region as datasets_potentialarea_pv_roof_region_*
+
+module rli_pv_wfr_region:
+ snakefile: "rli_pv_wfr_region/create.smk"
+ config: config["store"]["datasets"]["rli_pv_wfr_region"]
+use rule * from rli_pv_wfr_region as datasets_rli_pv_wfr_region_ *
+
+module technology_data:
+ snakefile: "technology_data/create.smk"
+ config: config["store"]["datasets"]["technology_data"]
+use rule * from technology_data as datasets_technology_data_ *
+
+module osm_buildings:
+ snakefile: "osm_buildings/create.smk"
+ config: config["store"]["datasets"]["osm_buildings"]
+use rule * from osm_buildings as datasets_osm_buildings_ *
+
+module emissions_region:
+ snakefile: "emissions_region/create.smk"
+ config: config["store"]["datasets"]["emissions_region"]
+use rule * from emissions_region as datasets_emissions_region_ *
+
+module app_captions:
+ snakefile: "app_captions/create.smk"
+ config: config["store"]["datasets"]["app_captions"]
+use rule * from app_captions as datasets_app_captions_ *
+
+module app_settings:
+ snakefile: "app_settings/create.smk"
+ config: config["store"]["datasets"]["app_settings"]
+use rule * from app_settings as datasets_app_settings_*
diff --git a/digipipe/store/datasets/osm_buildings/config.yml b/digipipe/store/datasets/osm_buildings/config.yml
new file mode 100644
index 00000000..aeacb2f8
--- /dev/null
+++ b/digipipe/store/datasets/osm_buildings/config.yml
@@ -0,0 +1,17 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# If true, the conditions from "tags" are used as filters (e.g.
+# ["building", "yes"] results in building=yes)
+# If false, the keys from "tags" are selected but no further conditions apply
+use_conditions: false
+
+# Tags/conditions to include (format: [tag, value]))
+tags:
+ [
+ ["building", "*"],
+ ]
+
+# Intermediate and output layer name
+layer_name: "osm_buildings"
diff --git a/digipipe/store/datasets/osm_buildings/create.smk b/digipipe/store/datasets/osm_buildings/create.smk
new file mode 100644
index 00000000..8f9109cf
--- /dev/null
+++ b/digipipe/store/datasets/osm_buildings/create.smk
@@ -0,0 +1,146 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import json
+import geopandas as gpd
+
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ create_tag_string_ogr
+)
+
+DATASET_PATH = get_abs_dataset_path(
+ "datasets", "osm_buildings", data_dir=True)
+
+rule extract_buildings:
+ """
+ Create layers from converted OSM file using requested tags per layer. Only
+ requested attributes are retained.
+ """
+ input:
+ get_abs_dataset_path(
+ "preprocessed", "osm_filtered", data_dir=True
+ ) / "germany-230101_filtered.osm.gpkg"
+ output: DATASET_PATH / "osm_buildings.gpkg"
+ params:
+ tags=create_tag_string_ogr(config["tags"]),
+ layer_name=config["layer_name"]
+ run:
+ conditions = (
+ params.tags['conditions']
+ if config["use_conditions"]
+ else ""
+ )
+ shell(
+ f"ogr2ogr -f GPKG -select {params.tags['tags']} "
+ f"{conditions} {output} {input} -nln {params.layer_name} "
+ f"multipolygons"
+ )
+
+rule create_centroids:
+ """
+ Create centroids for buildings and attach ground area (in sqm)
+ """
+ input: DATASET_PATH / "osm_buildings.gpkg"
+ output: DATASET_PATH / "osm_buildings_centroids.gpkg"
+ params:
+ layer_name=config["layer_name"]
+ run:
+ shell(
+ f"ogr2ogr -f GPKG -sql "
+ f"'SELECT ST_Centroid(geom) as geom, ST_Area(geom) AS area_sqm "
+ f"FROM {params.layer_name}' -dialect sqlite "
+ f"{output} {input} -nln '{params.layer_name}'"
+ )
+
+rule merge_with_region_file:
+ """
+ Merge centroids file with region file
+ """
+ input:
+ centroids=DATASET_PATH / "osm_buildings_centroids.gpkg",
+ region=rules.datasets_bkg_vg250_region_create.output
+ output: DATASET_PATH / "osm_buildings_centroids_regionmerge.gpkg"
+ params:
+ layer_name=config["layer_name"]
+ run:
+ shell(
+ f"ogrmerge.py -f GPKG -o {output} {input.centroids} {input.region} "
+ "-nln '{{DS_BASENAME}}'"
+ )
+
+rule intersect_with_region:
+ """
+ Intersect centroids with region
+ """
+ input: DATASET_PATH / "osm_buildings_centroids_regionmerge.gpkg"
+ output: DATASET_PATH / "osm_buildings_centroids_region.gpkg"
+ params:
+ layer_name=config["layer_name"]
+ run:
+ shell(
+ f"ogr2ogr -sql 'SELECT a.area_sqm, a.geom "
+ f"FROM osm_buildings_centroids AS a, bkg_vg250_region AS b "
+ f"WHERE ST_INTERSECTS(a.geom, b.geom)' -dialect sqlite "
+ f"{output} {input} -nln '{params.layer_name}'"
+ )
+
+rule calc_area_totals:
+ """
+ Calculate total ground area of buildings for region and country
+ """
+ input:
+ region=DATASET_PATH / "osm_buildings_centroids_region.gpkg",
+ country=DATASET_PATH / "osm_buildings_centroids.gpkg"
+ output:
+ region=DATASET_PATH / "osm_buildings_ground_area_region.gpkg",
+ country=DATASET_PATH/ "osm_buildings_ground_area_country.gpkg"
+ params:
+ layer_name=config["layer_name"]
+ run:
+ shell(
+ f"ogr2ogr -sql 'SELECT sum(area_sqm) AS area_sum_sqm "
+ f"FROM {params.layer_name}' -dialect sqlite "
+ f"{output.region} {input.region} -nln '{params.layer_name}'"
+ )
+ shell(
+ f"ogr2ogr -sql 'SELECT sum(area_sqm) AS area_sum_sqm "
+ f"FROM {params.layer_name}' -dialect sqlite "
+ f"{output.country} {input.country} -nln '{params.layer_name}'"
+ )
+
+rule calc_building_ground_area_share:
+ """
+ Calculate share of region's total building ground area in country's total
+ building ground area.
+ """
+ input:
+ region=DATASET_PATH / "osm_buildings_ground_area_region.gpkg",
+ country=DATASET_PATH/ "osm_buildings_ground_area_country.gpkg"
+ output:
+ DATASET_PATH / "osm_buildings_ground_area.json"
+ run:
+ area_region = round(
+ float(gpd.read_file(input.region).area_sum_sqm.sum())
+ )
+ area_country = round(
+ float(gpd.read_file(input.country).area_sum_sqm.sum())
+ )
+ area_share = round(area_region / area_country, 4)
+ print(
+ f"Share of region's total building ground area in country's total "
+ f"building ground area: {area_share}"
+ )
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(
+ {
+ "building_ground_area_country": area_country,
+ "building_ground_area_region": area_region,
+ "building_ground_area_share_region": area_share
+ },
+ f,
+ indent=4
+ )
diff --git a/digipipe/store/datasets/osm_buildings/data/.gitkeep b/digipipe/store/datasets/osm_buildings/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/osm_buildings/dataset.md b/digipipe/store/datasets/osm_buildings/dataset.md
new file mode 100644
index 00000000..f8645d35
--- /dev/null
+++ b/digipipe/store/datasets/osm_buildings/dataset.md
@@ -0,0 +1,19 @@
+# OpenStreetMap Gebäude
+
+OSM Gebäude aus [osm_filtered](../../preprocessed/osm_filtered/dataset.md)
+mittels OGR extrahieren und nach Tags (s. [config.yml](config.yml)) filtern.
+
+Ziel ist die Ermittlung des regionalen Anteils Gebäudegrundflächen an der
+gesamten Gebäudegrundfläche in Deutschland.
+
+Schritte:
+
+- Extraktion aller Gebäude in Deutschland --> `osm_buildings.gpkg`
+- Zentroide und Fläche je Gebäude erstellen --> `osm_buildings_centroids.gpkg`
+- Mit Region verschneiden --> `osm_buildings_centroids_region.gpkg`
+- Flächensumme berechnen --> `osm_buildings_ground_area_region.gpkg`,
+ `osm_buildings_ground_area_country.gpkg`
+- Regionalen Anteil berechnen --> `osm_buildings_ground_area_share_region.json`
+
+**Achtung:** Konvertierungs- und Extraktionsprozess benötigt ~15 GB
+Speicherplatz und kann viel Zeit in Anspruch nehmen.
diff --git a/digipipe/store/datasets/population_region/config.yml b/digipipe/store/datasets/population_region/config.yml
new file mode 100644
index 00000000..6a3b6e4a
--- /dev/null
+++ b/digipipe/store/datasets/population_region/config.yml
@@ -0,0 +1,29 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Population prognosis: years to be included from state prognosis
+# data on municipal level (empty list to deactivate).
+prognosis_fstate_munlevel:
+ years:
+ [
+ 2025,
+ 2030,
+ 2035
+ ]
+
+# Population prognosis: years to be included from country prognosis
+# data on state level (empty list to deactivate).
+prognosis_germany_districtlevel:
+ years:
+ [
+ 2040,
+ 2045
+ ]
+
+# Population extrapolation: years to be covered (empty list to deactivate)
+# Note: all available prognosis years will be used for the extrapolation,
+# not-requested prognosis years (above) will be simply dropped in the end.
+extrapolation:
+ years:
+ []
diff --git a/digipipe/store/datasets/population_region/create.smk b/digipipe/store/datasets/population_region/create.smk
new file mode 100644
index 00000000..d23114b1
--- /dev/null
+++ b/digipipe/store/datasets/population_region/create.smk
@@ -0,0 +1,34 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ PATH_TO_REGION_DISTRICTS_GPKG
+)
+
+DATASET_PATH = get_abs_dataset_path("datasets", "population_region")
+
+rule create:
+ """
+ Create full population dataset for region
+ """
+ input:
+ pop_history=expand(
+ get_abs_dataset_path("preprocessed", "destatis_gv") / "data" / "3112{year}_Auszug_GV.csv",
+ year=[2010, 2015, 2020, 2021, 2022]
+ ),
+ prognosis_fstate_munlevel=get_abs_dataset_path(
+ "preprocessed", "stala_st_pop_prog") / "data" /
+ "population_prognosis_st_muns.csv",
+ prognosis_germany_districtlevel=get_abs_dataset_path(
+ "preprocessed", "demandregio") / "data" / "dr_hh_population.csv",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ region_districts=PATH_TO_REGION_DISTRICTS_GPKG
+ output:
+ DATASET_PATH / "data" / "population.csv"
+ script:
+ DATASET_PATH / "scripts" / "create.py"
diff --git a/digipipe/store/datasets/population_region/data/.gitkeep b/digipipe/store/datasets/population_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/population_region/dataset.md b/digipipe/store/datasets/population_region/dataset.md
new file mode 100644
index 00000000..89f35d7e
--- /dev/null
+++ b/digipipe/store/datasets/population_region/dataset.md
@@ -0,0 +1,36 @@
+# Bevölkerungsentwicklung
+
+EinwohnerInnen je Gemeinde: Historische Daten und Prognosen
+
+## Historische Daten bis 2022
+
+Statistisches Bundesamt (Raw dataset:
+[destatis_gv](../../raw/destatis_gv/dataset.md))
+
+## Prognosen bis 2035
+
+Statistisches Landesamt Sachsen-Anhalt (Raw dataset:
+[stala_st_pop_prog](../../raw/stala_st_pop_prog/dataset.md)). Deaktivieren
+mittels entfernen der Zieljahre in [config.yml](config.yml) im Abschnitt
+`prognosis_fstate_munlevel`.
+
+Kann für andere Regionen auch durch DemandRegio (s.u.) ersetzt werden, die
+tatsächliche regionale Auflösung wird dadurch reduziert.
+
+## Prognosen bis 2045
+
+DemandRegio (Raw dataset: [demandregio](../../raw/demandregio/dataset.md))
+basierend auf der
+[14. koordinierten Bevölkerungsvorausberechnung](https://www.destatis.de/DE/Themen/Gesellschaft-Umwelt/Bevoelkerung/Bevoelkerungsvorausberechnung/aktualisierung-bevoelkerungsvorausberechnung.html)
+der Statistischen Ämter von Bund und Ländern. Diese Daten liegen auf
+Landkreisebene vor, daher erfolgt eine gleichmäßige Skalierung der
+dazugehörigen Gemeinden auf den jeweiligen Prognosewert.
+
+Deaktivieren mittels entfernen der Zieljahre in [config.yml](config.yml) im
+Abschnitt `prognosis_germany_districtlevel`.
+
+## Extrapolation
+
+Über 2045 hinaus wird lineare Extrapolation auf Basis der letzten beiden
+Prognosejahre unterstützt. Um diese zu aktivieren, müssen lediglich Zieljahre
+in die [config.yml](config.yml) im Abschnitt `extrapolation` eingetragen werden.
diff --git a/digipipe/store/datasets/population_region/scripts/create.py b/digipipe/store/datasets/population_region/scripts/create.py
new file mode 100644
index 00000000..512d1a11
--- /dev/null
+++ b/digipipe/store/datasets/population_region/scripts/create.py
@@ -0,0 +1,185 @@
+import geopandas as gpd
+import pandas as pd
+
+
+def process() -> None:
+ # pylint: disable=R0914
+ # Get muns and their NUTS3 code
+ muns = gpd.read_file(snakemake.input.region_muns)
+ districts = gpd.read_file(snakemake.input.region_districts)
+ muns = muns.merge(
+ districts[["id", "nuts"]].rename(columns={"id": "district_id"}),
+ left_on="district_id",
+ right_on="district_id",
+ )
+
+ # Historical data: load, get available years and filter by region
+ pop_history = pd.concat(
+ [
+ pd.read_csv(f, dtype={"ags": str}, index_col="ags")
+ for f in snakemake.input.pop_history
+ ],
+ axis=1,
+ )
+ pop_history = pop_history.loc[pop_history.index.isin(muns.ags)]
+ avail_years_history = pop_history.columns.astype(int).to_list()
+ print(f"Historical years: {avail_years_history}")
+
+ # Prognosis: load state data on municipal level and filter by region
+ pop_prognosis_years_mun = snakemake.config["prognosis_fstate_munlevel"][
+ "years"
+ ]
+ if len(pop_prognosis_years_mun) > 0:
+ pop_prognosis_mun = pd.read_csv(
+ snakemake.input.prognosis_fstate_munlevel,
+ dtype={"ags": str},
+ index_col="ags",
+ ).round()
+ pop_prognosis_mun = pop_prognosis_mun.loc[
+ pop_prognosis_mun.index.isin(muns.ags)
+ ]
+ avail_years_prognosis = pop_prognosis_mun.columns.astype(int).to_list()
+ print(
+ f"Prognosis years (state data on municipal level): "
+ f"{pop_prognosis_years_mun}"
+ )
+ pop_reference = pop_prognosis_mun
+ pop_reference_years = pop_prognosis_years_mun
+ else:
+ pop_reference = pop_history
+ pop_reference_years = avail_years_history
+ print(
+ "No years for prognosis with state data on municipal level "
+ "provided, skipping..."
+ )
+
+ # Prognosis: load country data on district (NUTS 3) level
+ pop_prognosis_years_district = snakemake.config[
+ "prognosis_germany_districtlevel"
+ ]["years"]
+ if len(pop_prognosis_years_district) > 0:
+ pop_prognosis_district = pd.read_csv(
+ snakemake.input.prognosis_germany_districtlevel,
+ index_col="nuts3",
+ )
+ pop_prognosis_district = pop_prognosis_district.loc[muns.nuts.unique()][
+ [str(y) for y in pop_prognosis_years_district]
+ ]
+ avail_years_prognosis = pop_prognosis_district.columns.astype(
+ int
+ ).to_list()
+ print(
+ f"Prognosis years (country data on district level): "
+ f"{pop_prognosis_years_district}"
+ )
+
+ # Merge prognoses: create municipal prognosis from NUTS 3 level data by
+ # using municipal shares from the last available state's municipal
+ # prognosis.
+ pop_reference_lastyear = pd.concat(
+ [
+ pop_reference[str(pop_reference_years[-1])],
+ muns[["ags", "nuts"]].set_index("ags"),
+ ],
+ axis=1,
+ )
+ pop_reference_lastyear = (
+ pop_reference_lastyear.assign(
+ share=(
+ pop_reference_lastyear[str(pop_reference_years[-1])]
+ / pop_reference_lastyear.groupby("nuts")[
+ str(pop_reference_years[-1])
+ ].transform("sum")
+ )
+ )
+ .drop(columns=[str(pop_reference_years[-1])])
+ .reset_index()
+ )
+ pop_prognosis_district = pop_reference_lastyear.merge(
+ pop_prognosis_district.reset_index(),
+ left_on="nuts",
+ right_on="nuts3",
+ ).set_index("ags")
+ for year in pop_prognosis_years_district:
+ pop_reference[str(year)] = (
+ pop_prognosis_district["share"]
+ .mul(pop_prognosis_district[str(year)])
+ .round()
+ )
+ else:
+ print(
+ "No years for prognosis with country data on district level "
+ "provided, skipping..."
+ )
+
+ # Extrapolate population linearly for years from config
+ extrapol_years = snakemake.config["extrapolation"]["years"]
+ if len(extrapol_years) > 0:
+ print(f"Extrapolation years: {extrapol_years}")
+ for year in extrapol_years:
+ # Extrapolate using the last 2 available years
+ year_delta_base = (
+ avail_years_prognosis[-1] - avail_years_prognosis[-2]
+ )
+ year_delta_extrapol = year - avail_years_prognosis[-1]
+ pop_reference[str(year)] = (
+ pop_reference[str(avail_years_prognosis[-1])]
+ + (
+ pop_reference[str(avail_years_prognosis[-1])]
+ - pop_reference[str(avail_years_prognosis[-2])]
+ )
+ / year_delta_base
+ * year_delta_extrapol
+ )
+ else:
+ print("No extrapolation years provided, skipping extrapolation...")
+
+ # Drop not requested years
+ pop_reference.drop(
+ columns=[
+ c
+ for c in pop_reference.columns
+ if int(c)
+ not in (
+ avail_years_history
+ + pop_reference_years
+ + pop_prognosis_years_district
+ + extrapol_years
+ )
+ ],
+ inplace=True,
+ )
+
+ # Extra-include historic data depending on if first prognosis was used
+ if not all(
+ str(year) in pop_reference.columns for year in avail_years_history
+ ):
+ population = pd.concat([pop_history, pop_reference], axis=1)
+ else:
+ population = pop_reference
+
+ # Add municipality_id and data origin
+ population = (
+ pd.concat(
+ [muns.set_index("ags")["id"].rename("municipality_id"), population],
+ axis=1,
+ )
+ .sort_index()
+ .set_index("municipality_id", drop=True)
+ )
+ population.columns = pd.MultiIndex.from_arrays(
+ [
+ population.columns,
+ len(avail_years_history) * ["historic"]
+ + (len(pop_prognosis_years_mun) + len(pop_prognosis_years_district))
+ * ["prognosis"]
+ + len(extrapol_years) * ["extrapolation"],
+ ],
+ names=("year", "type"),
+ )
+
+ population.to_csv(snakemake.output[0])
+
+
+if __name__ == "__main__":
+ process()
diff --git a/digipipe/store/datasets/potentialarea_pv_ground_region/config.yml b/digipipe/store/datasets/potentialarea_pv_ground_region/config.yml
new file mode 100644
index 00000000..99e76cbd
--- /dev/null
+++ b/digipipe/store/datasets/potentialarea_pv_ground_region/config.yml
@@ -0,0 +1,15 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Area names (which identify files)
+areas:
+ [
+ "agriculture_lfa-off_region",
+ "road_railway_region",
+ ]
+
+# PV targets split: share of pv_ground
+# Caution: when adjusting this value, the pv_roof_share in dataset
+# potentialarea_pv_roof_region must be adjusted accordingly!
+pv_ground_share: 0.5
diff --git a/digipipe/store/datasets/potentialarea_pv_ground_region/create.smk b/digipipe/store/datasets/potentialarea_pv_ground_region/create.smk
new file mode 100644
index 00000000..107879e1
--- /dev/null
+++ b/digipipe/store/datasets/potentialarea_pv_ground_region/create.smk
@@ -0,0 +1,218 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import json
+import re
+import geopandas as gpd
+import pandas as pd
+from pathlib import Path
+
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.data_io import load_json
+from digipipe.scripts.geo import (
+ overlay,
+ convert_to_multipolygon,
+ write_geofile
+)
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG
+)
+
+DATASET_PATH = get_abs_dataset_path(
+ "datasets", "potentialarea_pv_ground_region", data_dir=True
+)
+
+rule overlay_muns:
+ """
+ Overlay potential area with municipalities
+ """
+ input:
+ area=get_abs_dataset_path(
+ "datasets", "rli_pv_wfr_region", data_dir=True) / "{file}",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG
+ output:
+ area=DATASET_PATH / "{file}"
+ run:
+ data = gpd.read_file(input.area)
+ data = overlay(
+ gdf=data,
+ gdf_overlay=gpd.read_file(input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ write_geofile(
+ gdf=convert_to_multipolygon(data),
+ file=output.area,
+ )
+
+rule create_area_stats_muns:
+ """
+ Create stats on pv potential areas per mun
+ """
+ input:
+ area=expand(
+ DATASET_PATH / "potentialarea_pv_{area}.gpkg",
+ area=config["areas"],
+ ),
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ output: DATASET_PATH / "potentialarea_pv_ground_area_stats_muns.csv"
+ run:
+ print("PV ground potential area stats:")
+ muns = gpd.read_file(input.region_muns)
+ area_dict = {}
+
+ # Calc areas per area type file
+ for file in input.area:
+ area_name = re.findall(
+ "potentialarea_pv_(.*).gpkg",
+ Path(file).name,
+ )[0]
+ data = gpd.read_file(file)
+ data["area_km2"] = data.area / 1e6
+ area_km2 = data[
+ ["municipality_id", "area_km2"]
+ ].groupby("municipality_id").sum()
+
+ # Set area of non-occurring muns to 0
+ area_km2 = area_km2.reindex(muns.id, fill_value=0)
+ area_dict[area_name] = area_km2.to_dict()["area_km2"]
+ print(
+ f" Total area for {area_name}: "
+ f"{round(float(area_km2.sum()), 1)} sqm"
+ )
+
+ area_df = pd.DataFrame(area_dict)
+ area_df.index.name="municipality_id"
+ area_df.to_csv(output[0])
+
+rule create_potarea_shares:
+ """
+ Calc shares of actual potential areas in total potential areas (per type)
+ """
+ input:
+ potarea_pv_road_railway=get_abs_dataset_path(
+ "datasets", "rli_pv_wfr_region", data_dir=True
+ ) / "potentialarea_pv_road_railway_region.gpkg",
+ road_railway_500m=get_abs_dataset_path(
+ "datasets", "rli_pv_wfr_region", data_dir=True
+ ) / "road_railway-500m_region.gpkg",
+ potarea_pv_agri=get_abs_dataset_path(
+ "datasets", "rli_pv_wfr_region", data_dir=True
+ ) / "potentialarea_pv_agriculture_lfa-off_region.gpkg",
+ soil_quality_low=get_abs_dataset_path(
+ "datasets", "rli_pv_wfr_region", data_dir=True
+ ) / "soil_quality_low_region.gpkg",
+
+ output: DATASET_PATH / "potentialarea_pv_ground_area_shares.json"
+ run:
+ area_dict = {
+ "road_railway": round(
+ gpd.read_file(input.potarea_pv_road_railway).area.sum() /
+ gpd.read_file(input.road_railway_500m).area.sum(),
+ 3
+ ),
+ "agri": round(
+ gpd.read_file(input.potarea_pv_agri).area.sum() /
+ gpd.read_file(input.soil_quality_low).area.sum(),
+ 3
+ ),
+ }
+
+ # Dump
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(area_dict, f, indent=4)
+
+rule regionalize_state_targets:
+ """
+ Calculate PV ground targets of region
+ """
+ input:
+ potarea_pv_road_railway=get_abs_dataset_path(
+ "preprocessed", "rli_pv_wfr", data_dir=True
+ ) / "potentialarea_pv_road_railway.gpkg",
+ potarea_pv_agri=get_abs_dataset_path(
+ "preprocessed", "rli_pv_wfr", data_dir=True
+ ) / "potentialarea_pv_agriculture_lfa-off.gpkg",
+ potarea_pv_road_railway_region=get_abs_dataset_path(
+ "datasets", "rli_pv_wfr_region", data_dir=True
+ ) / "potentialarea_pv_road_railway_region.gpkg",
+ potarea_pv_agri_region=get_abs_dataset_path(
+ "datasets", "rli_pv_wfr_region", data_dir=True
+ ) / "potentialarea_pv_agriculture_lfa-off_region.gpkg",
+ el_capacity_targets=get_abs_dataset_path(
+ "preprocessed", "bmwk_long_term_scenarios"
+ ) / "data" / "T45-Strom_electricity_installed_power_reformatted.csv",
+ tech_data=get_abs_dataset_path(
+ "datasets", "technology_data", data_dir=True
+ ) / "technology_data.json"
+ output:
+ DATASET_PATH / "potentialarea_pv_ground_regionalized_targets.json"
+ run:
+ potarea_pv_rr = gpd.read_file(input.potarea_pv_road_railway).to_crs(
+ GLOBAL_CONFIG["global"]["geodata"]["crs"]
+ ).area.sum()
+ potarea_pv_agri = gpd.read_file(input.potarea_pv_agri).to_crs(
+ GLOBAL_CONFIG["global"]["geodata"]["crs"]
+ ).area.sum()
+ potarea_pv_rr_region = gpd.read_file(
+ input.potarea_pv_road_railway_region).area.sum()
+ potarea_pv_agri_region = gpd.read_file(
+ input.potarea_pv_agri_region).area.sum()
+ tech_data = load_json(input.tech_data)
+
+ # Power target from longterm scenario
+ targets = pd.read_csv(input.el_capacity_targets, index_col="year")
+ target_cap = targets.loc[
+ targets.technology == "pv"
+ ].loc[2045].capacity * 1e3 * config.get("pv_ground_share")
+ targets_region = (
+ target_cap * (
+ (potarea_pv_rr_region + potarea_pv_agri_region) /
+ (potarea_pv_rr + potarea_pv_agri)
+ )
+ )
+
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(
+ {
+ # Power targets (disaggregated)
+ "target_power_total": round(
+ target_cap * (
+ (potarea_pv_rr_region + potarea_pv_agri_region) /
+ (potarea_pv_rr + potarea_pv_agri)
+ )
+ ),
+ "target_power_road_railway": round(
+ target_cap * potarea_pv_rr_region / (
+ potarea_pv_rr + potarea_pv_agri)
+ ),
+ "target_power_agri": round(
+ target_cap * potarea_pv_agri_region / (
+ potarea_pv_rr + potarea_pv_agri)
+ ),
+ # Areas targets (from power targets)
+ "target_area_total": round(
+ target_cap * (
+ (potarea_pv_rr_region + potarea_pv_agri_region) /
+ (potarea_pv_rr + potarea_pv_agri)
+ ) / tech_data["power_density"]["pv_ground"]
+ ,2
+ ),
+ "target_area_road_railway": round(
+ target_cap * potarea_pv_rr_region / (
+ potarea_pv_rr + potarea_pv_agri
+ ) / tech_data["power_density"]["pv_ground"]
+ ,2
+ ),
+ "target_area_agri": round(
+ target_cap * potarea_pv_agri_region / (
+ potarea_pv_rr + potarea_pv_agri
+ ) / tech_data["power_density"]["pv_ground"]
+ ,2
+ ),
+ },
+ f,
+ indent=4
+ )
diff --git a/digipipe/store/datasets/potentialarea_pv_ground_region/data/.gitkeep b/digipipe/store/datasets/potentialarea_pv_ground_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/potentialarea_pv_ground_region/dataset.md b/digipipe/store/datasets/potentialarea_pv_ground_region/dataset.md
new file mode 100644
index 00000000..e0ba2a16
--- /dev/null
+++ b/digipipe/store/datasets/potentialarea_pv_ground_region/dataset.md
@@ -0,0 +1,59 @@
+# Potenzialgebiete PV-Freiflächen
+
+## Potenzialflächen
+
+Potenzialgebiete für die Errichtung von PV-Freiflächenanlagen aus dem
+[PV- und Windflächenrechner](https://www.agora-energiewende.de/service/pv-und-windflaechenrechner/)
+(s. Datensatz [rli_pv_wfr](../../raw/rli_pv_wfr/dataset.md)).
+
+Die Potenzialflächen bilden jene Flächen ab, die für die Nutzung durch
+Freiflächen-Photovoltaikanlagen grundsätzlich zur Verfügung stehen. Sie
+orientieren sich an der aktuellen Förderkulisse und wurden anhand des
+Flächenumfangs sowie den verfügbaren Geodaten ausgewählt: Von den in §37 EEG
+2021 definierten Flächen werden Flächen nach §37 Absatz 1 Nummer 2 Buchstaben c,
+h und i berücksichtigt (für Details zur Methodik siehe
+[methodisches Begleitdokument](https://zenodo.org/record/6794558) zum PV- und
+Windflächenrechner).
+
+Dateien:
+
+- Freiflächen-PV auf Acker- und Grünlandflächen mit geringer Bodengüte (Soil
+ Quality Rating (SQR) < 40): `potentialarea_pv_agriculture_lfa-off_region.gpkg`
+- Potenzialflächen für Freiflächen-PV entlang von Bundesautobahnen und
+ Schienenwegen (500m-Streifen): `potentialarea_pv_road_railway_region.gpkg`
+
+## Statistische Auswertung
+
+Die Flächen werden mit den Gemeindegrenzen verschnitten und den Gemeinden
+zugeordnet. Je Gemeinde und obigem Flächentyp/Datei wird eine Flächensumme (in
+km²) berechnet, siehe `potentialarea_pv_ground_area_stats_muns.csv`. Die
+Gemeinden werden über den Schlüssel `municipality_id` (vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md))
+identifiziert.
+
+Des Weiteren werden die Flächenanteile der verfügbaren Potenzialgebiete - deren
+Nutzung nur eingeschränkt möglich ist (z.B. durch Naturschutzgebieten etc.) -
+gegenüber den gesamten Potenzialgebiete (für die Parametrierung der Regler) nach
+`potentialarea_pv_ground_area_shares.json` exportiert.
+
+## Ausbauziele
+
+Es werden PV-Ausbauziele für die Region berechnet, indem die Bundesziele aus den
+[BMWK Langfristszenarien](../../preprocessed/bmwk_long_term_scenarios/dataset.md)
+i.H.v. 428 GW
+([§4 EEG 2023](https://www.gesetze-im-internet.de/eeg_2014/__4.html): 400 GW)
+anhand der regional verfügbaren Potenzialflächen disaggregiert werden. Hierzu
+wird der Anteil der Flächensumme der beiden o.g. Flächentypen an den bundesweit
+verfügbaren Flächen (Datensatz [rli_pv_wfr](../../raw/rli_pv_wfr/dataset.md))
+berechnet. Da in den o.g. Ausbauzielen nicht zwischen Freiflächen- und
+Aufdach-PV unterschieden wird, wird ein Verhältnis von 50:50 angenommen, d.h.
+bundesweit 214 GW auf Freiflächen-PV entfallen.
+
+Es ergeben sich folgende Flächen- und Leistungsanteile:
+
+Gesamt: 0.38 % (819 MW)
+
+- Entlang von BAB und Schienenwegen: 0.13 % (278 MW)
+- Acker- und Grünlandflächen mit geringer Bodengüte: 0.25 % (541 MW)
+
+Ergebnisse in `potentialarea_pv_ground_regionalized_targets.json`
diff --git a/digipipe/store/datasets/potentialarea_pv_ground_region/scripts/create.py b/digipipe/store/datasets/potentialarea_pv_ground_region/scripts/create.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/potentialarea_pv_roof_region/config.yml b/digipipe/store/datasets/potentialarea_pv_roof_region/config.yml
new file mode 100644
index 00000000..671e4259
--- /dev/null
+++ b/digipipe/store/datasets/potentialarea_pv_roof_region/config.yml
@@ -0,0 +1,8 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# PV targets split: share of pv_roof
+# Caution: when adjusting this value, the pv_ground_share in dataset
+# potentialarea_pv_ground_region must be adjusted accordingly!
+pv_roof_share: 0.5
diff --git a/digipipe/store/datasets/potentialarea_pv_roof_region/create.smk b/digipipe/store/datasets/potentialarea_pv_roof_region/create.smk
new file mode 100644
index 00000000..c5ad8a2e
--- /dev/null
+++ b/digipipe/store/datasets/potentialarea_pv_roof_region/create.smk
@@ -0,0 +1,217 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import json
+import re
+import geopandas as gpd
+import pandas as pd
+from digipipe.scripts.data_io import load_json
+from digipipe.scripts.geo import (
+ overlay,
+ convert_to_multipolygon,
+ write_geofile
+)
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG
+)
+
+DATASET_PATH = get_abs_dataset_path(
+ "datasets", "potentialarea_pv_roof_region", data_dir=True
+)
+
+rule overlay_muns:
+ """
+ Overlay potential area with municipalities
+ """
+ input:
+ area=get_abs_dataset_path(
+ "preprocessed", "rpg_abw_pv_roof_potential", data_dir=True
+ ) / "rpg_abw_pv_roof_potential.gpkg",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG
+ output:
+ area=DATASET_PATH / "potentialarea_pv_roof_region.gpkg"
+ run:
+ data = gpd.read_file(input.area)
+ data = data.assign(
+ historic_preservation=data.historic_preservation.fillna(
+ False).replace({"true": True})
+ )
+ data = overlay(
+ gdf=data,
+ gdf_overlay=gpd.read_file(input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ gdf_use_centroid=True
+ )
+ write_geofile(
+ gdf=convert_to_multipolygon(data),
+ file=output.area,
+ )
+
+rule create_area_stats_muns:
+ """
+ Create stats on PV roof potentials per mun for 1) all and 2) all but
+ historic buildings.
+ """
+ input:
+ area=DATASET_PATH / "potentialarea_pv_roof_region.gpkg",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ output:
+ total=DATASET_PATH / "potentialarea_pv_roof_area_stats_muns.csv",
+ wo_historic=(
+ DATASET_PATH /
+ "potentialarea_pv_roof_wo_historic_area_stats_muns.csv"
+ )
+ run:
+ print("PV roof potential area stats:")
+ muns = gpd.read_file(input.region_muns)
+ potential_all = gpd.read_file(input.area).fillna(0)
+
+ # define columns
+ cols_base = [
+ "municipality_id", "historic_preservation", "building_area_sqm"
+ ]
+ orientation_suffix = ["south", "north", "east", "west", "flat"]
+ cols_power = [
+ f"installable_power_kw_{orient}"
+ for orient in orientation_suffix
+ ]
+ cols_power_new = {
+ c.replace("_kw_", "_"): (c, "sum") for c in cols_power
+ }
+ cols_energy = [
+ f"energy_annual_mwh_{orient}"
+ for orient in orientation_suffix
+ ]
+ cols_energy_new = {
+ c.replace("_mwh_","_"): (c, "sum") for c in cols_energy
+ }
+
+ # aggregate per mun
+ agg_cols = dict(
+ roof_count=("building_area_sqm", "count"),
+ building_area_sqm=("building_area_sqm", "sum"),
+ historic_preservation_count=("historic_preservation", "sum"),
+ **cols_power_new,
+ **cols_energy_new
+ )
+ potential_wo_historic = potential_all.copy()
+ potential_wo_historic = potential_wo_historic.loc[
+ ~potential_wo_historic.historic_preservation
+ ]
+
+ for df, file in zip(
+ [potential_all, potential_wo_historic],
+ [output.total, output.wo_historic]
+ ):
+ df = df[
+ cols_base + cols_power + cols_energy
+ ].groupby("municipality_id").agg(**agg_cols)
+ # kW -> MW
+ df[[_ for _ in cols_power_new.keys()]] = df[
+ cols_power_new.keys()].div(1e3)
+ # Produce totals
+ df = df.assign(
+ installable_power_total=df[
+ [_ for _ in cols_power_new.keys()]].sum(axis=1),
+ energy_annual_total=df[
+ [_ for _ in cols_energy_new.keys()]].sum(axis=1),
+ )
+ # Dump
+ df.to_csv(file)
+
+rule create_relative_deployment_stats_muns:
+ """
+ Create stats on how much of the theoretically installable PV rooftop
+ potential is used per mun for 1) all and 2) all but historic buildings.
+ """
+ input:
+ area_stats_total=(
+ rules.datasets_potentialarea_pv_roof_region_create_area_stats_muns.output.total
+ ),
+ area_stats_wo_historic=(
+ rules.datasets_potentialarea_pv_roof_region_create_area_stats_muns.output.wo_historic
+ ),
+ unit_stats=(
+ rules.datasets_bnetza_mastr_pv_roof_region_create_power_stats_muns.output[0]
+ )
+ output:
+ total=DATASET_PATH / "potentialarea_pv_roof_deployment_stats_muns.csv",
+ wo_historic=(
+ DATASET_PATH/
+ "potentialarea_pv_roof_wo_historic_deployment_stats_muns.csv"
+ )
+ run:
+ orientation_suffix = ["south", "north", "east", "west", "flat"]
+ cols_power = [
+ f"installable_power_{orient}"
+ for orient in orientation_suffix
+ ]
+ pv_installed = pd.read_csv(
+ input.unit_stats,
+ usecols=["municipality_id", "capacity_net"],
+ index_col="municipality_id",
+ )
+
+ for file_in, file_out in zip(
+ [input.area_stats_total, input.area_stats_wo_historic],
+ [output.total, output.wo_historic]
+ ):
+ pv_potential = pd.read_csv(
+ file_in,
+ usecols=["municipality_id"] + cols_power,
+ index_col="municipality_id",
+ )
+ pv_potential = pd.DataFrame(
+ pv_potential.sum(axis=1),columns=["installable_power"]
+ )
+ pv_deployed = pd.concat(
+ [pv_installed.capacity_net, pv_potential.installable_power],
+ axis=1,
+ )
+ pv_deployed = pv_deployed.assign(
+ relative_deployment=(
+ pv_installed.capacity_net.div(
+ pv_potential.installable_power
+ )
+ )
+ )
+ pv_deployed.to_csv(file_out)
+
+rule regionalize_state_targets:
+ """
+ Calculate PV roof targets of region
+ """
+ input:
+ osm_buildings_stats=get_abs_dataset_path(
+ "datasets", "osm_buildings", data_dir=True
+ ) / "osm_buildings_ground_area.json",
+ el_capacity_targets=get_abs_dataset_path(
+ "preprocessed", "bmwk_long_term_scenarios"
+ ) / "data" / "T45-Strom_electricity_installed_power_reformatted.csv",
+ output:
+ DATASET_PATH / "potentialarea_pv_roof_regionalized_targets.json"
+ run:
+ osm_buildings_stats = load_json(input.osm_buildings_stats)
+
+ # Power target from longterm scenario
+ targets = pd.read_csv(input.el_capacity_targets, index_col="year")
+
+ target_cap = targets.loc[
+ targets.technology == "pv"
+ ].loc[2045].capacity * 1e3 * config.get("pv_roof_share")
+
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(
+ {
+ # Power targets (disaggregated)
+ "target_power_total": round(
+ target_cap *
+ osm_buildings_stats["building_ground_area_share_region"]
+ ),
+ },
+ f,
+ indent=4
+ )
diff --git a/digipipe/store/datasets/potentialarea_pv_roof_region/data/.gitkeep b/digipipe/store/datasets/potentialarea_pv_roof_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/potentialarea_pv_roof_region/dataset.md b/digipipe/store/datasets/potentialarea_pv_roof_region/dataset.md
new file mode 100644
index 00000000..44ca28a3
--- /dev/null
+++ b/digipipe/store/datasets/potentialarea_pv_roof_region/dataset.md
@@ -0,0 +1,42 @@
+# Dachflächenpotenzial PV-Aufdachanlagen in ABW
+
+Abschätzung der installierten Leistung und des Ertrags von PV-Aufdachanlagen in
+Anhalt-Bitterfeld-Wittenberg der Regionalen Planungsgemeinschaft aus Datensatz
+[rpg_abw_pv_roof_potential](../../raw/rpg_abw_pv_roof_potential/dataset.md).
+
+Die Gebäudezentroide werden mit den Gemeindegrenzen verschnitten und den
+Gemeinden zugeordnet.
+Ergebnisdaten:
+
+- Alle Gebäude: `potentialarea_pv_roof_area_stats_muns.csv`
+- Alle nicht denkmalgeschützten Gebäude:
+ `potentialarea_pv_roof_wo_historic_area_stats_muns.csv`
+
+Des Weiteren wird je Gemeinde der relative Anteil der bereits installierten
+Anlagenleistung an der theoretisch installierbaren Leistung (bei
+100% Dachnutzung) berechnet.
+Ergebnisdaten:
+
+- Alle Gebäude: `potentialarea_pv_roof_deployment_stats_muns.csv`
+- Alle nicht denkmalgeschützten Gebäude:
+ `potentialarea_pv_roof_wo_historic_deployment_stats_muns.csv`
+
+Die Gemeinden werden über den Schlüssel `municipality_id` (vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md))
+identifiziert.
+
+## Ausbauziele
+
+Es werden PV-Ausbauziele für die Region berechnet, indem die Bundesziele aus den
+[BMWK Langfristszenarien](../../preprocessed/bmwk_long_term_scenarios/dataset.md)
+i.H.v. 428 GW
+([§4 EEG 2023](https://www.gesetze-im-internet.de/eeg_2014/__4.html): 400 GW)
+anhand der Gebäudegrundflächen disaggregiert werden. Hierzu wird der Anteil der
+Gebäudegrundflächen in der Region an der bundesweiten Gebäudegrundflächen
+berechnet (s. Datensatz [osm_buildings](../osm_buildings/dataset.md)) und die
+Ziele linear skaliert. Da in den o.g. Ausbauzielen nicht zwischen Freiflächen-
+und Aufdach-PV unterschieden wird, wird ein Verhältnis von 50:50 angenommen,
+d.h. bundesweit 214 GW auf Aufdach-PV entfallen.
+
+Der Anteil beträgt 0,62 % und das Leistungsziel damit 1327 MW, s.
+`potentialarea_pv_roof_regionalized_targets.json`.
diff --git a/digipipe/store/datasets/potentialarea_pv_roof_region/scripts/create.py b/digipipe/store/datasets/potentialarea_pv_roof_region/scripts/create.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/potentialarea_wind_region/config.yml b/digipipe/store/datasets/potentialarea_wind_region/config.yml
new file mode 100644
index 00000000..79292e6f
--- /dev/null
+++ b/digipipe/store/datasets/potentialarea_wind_region/config.yml
@@ -0,0 +1,24 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Area names (which identify files from preprocessing)
+areas:
+ [
+ "stp_2018_vreg",
+ "stp_2027_repowering",
+ "stp_2027_search_area_forest_area",
+ "stp_2027_search_area_open_area",
+ "stp_2027_vr"
+ ]
+
+# Attributes' captions for the app
+captions:
+ {"name_1": "Typ",
+ "name_2": "Lage",
+ "name_3": "ID/Nr.",
+ "name_4": "Anmerkung",
+ "plan_status": "Plan Status",
+ "decision_date": "Beschlussdatum",
+ "authorization_date": "Genehmigungsdatum",
+ "in_force_date": "Datum des Inkrafttretens"}
diff --git a/digipipe/store/datasets/potentialarea_wind_region/create.smk b/digipipe/store/datasets/potentialarea_wind_region/create.smk
new file mode 100644
index 00000000..2463973a
--- /dev/null
+++ b/digipipe/store/datasets/potentialarea_wind_region/create.smk
@@ -0,0 +1,103 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import json
+import re
+import geopandas as gpd
+import pandas as pd
+from pathlib import Path
+from digipipe.scripts.geo import (
+ overlay,
+ convert_to_multipolygon,
+ write_geofile
+)
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ PATH_TO_REGION_MUNICIPALITIES_GPKG
+)
+
+DATASET_PATH = get_abs_dataset_path(
+ "datasets", "potentialarea_wind_region", data_dir=True
+)
+
+rule overlay_muns:
+ """
+ Overlay potential area with municipalities
+ """
+ input:
+ area=get_abs_dataset_path(
+ "preprocessed", "rpg_abw_regional_plan") / "data" / "{file}.gpkg",
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG
+ output:
+ area=DATASET_PATH / "potentialarea_wind_{file}.gpkg"
+ run:
+ data = gpd.read_file(input.area)
+ data = overlay(
+ gdf=data,
+ gdf_overlay=gpd.read_file(input.region_muns),
+ retain_rename_overlay_columns={"id": "municipality_id"},
+ )
+ write_geofile(
+ gdf=convert_to_multipolygon(data),
+ file=output.area,
+ )
+
+rule create_area_stats_muns:
+ """
+ Create stats on wind potential areas per mun
+ """
+ input:
+ area=expand(
+ DATASET_PATH / "potentialarea_wind_{area}.gpkg",
+ area=config["areas"],
+ ),
+ region_muns=PATH_TO_REGION_MUNICIPALITIES_GPKG,
+ output: DATASET_PATH / "potentialarea_wind_area_stats_muns.csv"
+ run:
+ print("Wind potential area stats:")
+ muns = gpd.read_file(input.region_muns)
+ area_dict = {}
+
+ # Calc areas per area type file
+ for file in input.area:
+ area_name = re.findall(
+ "potentialarea_wind_(.*).gpkg",
+ Path(file).name,
+ )[0]
+ data = gpd.read_file(file)
+ data["area_km2"] = data.area / 1e6
+ area_km2 = data[
+ ["municipality_id", "area_km2"]
+ ].groupby("municipality_id").sum()
+
+ # Set area of non-occurring muns to 0
+ area_km2 = area_km2.reindex(muns.id, fill_value=0)
+ area_dict[area_name] = area_km2.to_dict()["area_km2"]
+ print(
+ f" Total area for {area_name}: "
+ f"{round(float(area_km2.sum()), 1)} sqm"
+ )
+
+ area_df = pd.DataFrame(area_dict)
+ area_df.index.name="municipality_id"
+ area_df.to_csv(output[0])
+
+rule create_captions:
+ """
+ Create attribute captions for app
+ """
+ input: rules.datasets_potentialarea_wind_region_create_area_stats_muns.input.area
+ output: DATASET_PATH / "potentialarea_wind_attribute_captions.json"
+ run:
+ captions = {
+ "datasets_caption_map": {
+ Path(f).stem: "potentialarea_wind" for f in input
+ },
+ "captions": {
+ "potentialarea_wind": config["captions"]
+ }
+ }
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(captions, f, indent=4)
diff --git a/digipipe/store/datasets/potentialarea_wind_region/data/.gitkeep b/digipipe/store/datasets/potentialarea_wind_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/potentialarea_wind_region/dataset.md b/digipipe/store/datasets/potentialarea_wind_region/dataset.md
new file mode 100644
index 00000000..1200da2b
--- /dev/null
+++ b/digipipe/store/datasets/potentialarea_wind_region/dataset.md
@@ -0,0 +1,29 @@
+# Potenzialgebiete Windenergie
+
+Potenzialgebiete für die Errichtung von Windenergieanlagen, basierend auf den
+Teilplänen Wind der Regionalen Planungsgemeinschaft Anhalt-Bitterfeld-Wittenberg
+aus
+[rpg_abw_regional_plan](../../preprocessed/rpg_abw_regional_plan/dataset.md).
+
+Dateien:
+
+- STP Wind 2018 - Vorrang-/Eignungsgebiete:
+ `potentialarea_wind_stp_2018_vreg.gpkg`
+- STP Wind 2027 - Planabsicht Vorranggebiete:
+ `potentialarea_wind_stp_2027_vr.gpkg`
+- STP Wind 2027 - Planabsicht Repoweringgebiete:
+ `potentialarea_wind_stp_2027_repowering.gpkg`
+- STP Wind 2027 - Suchraum Wald:
+ `potentialarea_wind_stp_2027_search_area_forest_area.gpkg`
+- STP Wind 2027 - Suchraum Offenland:
+ `potentialarea_wind_stp_2027_search_area_open_area.gpkg`
+
+Die darin verwendeten Attributtexte werden in die Datei
+`potentialarea_wind_attribute_captions.json` exportiert.
+
+Die Flächen werden mit den Gemeindegrenzen verschnitten und den Gemeinden
+zugeordnet. Je Gemeinde und obigem Flächentyp/Datei wird eine Flächensumme (in
+km²) berechnet, siehe `potentialarea_wind_area_stats_muns.csv`. Die Gemeinden
+werden über den Schlüssel `municipality_id` (vgl.
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md))
+identifiziert.
diff --git a/digipipe/store/datasets/potentialarea_wind_region/scripts/create.py b/digipipe/store/datasets/potentialarea_wind_region/scripts/create.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/renewable_feedin/config.yml b/digipipe/store/datasets/renewable_feedin/config.yml
new file mode 100644
index 00000000..17d3af5a
--- /dev/null
+++ b/digipipe/store/datasets/renewable_feedin/config.yml
@@ -0,0 +1,3 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
diff --git a/digipipe/store/datasets/renewable_feedin/create.smk b/digipipe/store/datasets/renewable_feedin/create.smk
new file mode 100644
index 00000000..362223bd
--- /dev/null
+++ b/digipipe/store/datasets/renewable_feedin/create.smk
@@ -0,0 +1,24 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import pandas as pd
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("datasets", "renewable_feedin")
+
+rule normalize_feedin_timeseries:
+ """
+ Normalize feedin timeseries and drop time index
+ """
+ input:
+ get_abs_dataset_path(
+ "raw", "renewables.ninja_feedin") / "data" /
+ "{tech}_feedin_timeseries.csv"
+ output:
+ DATASET_PATH / "data" / "{tech}_feedin_timeseries.csv",
+ run:
+ feedin_timeseries = pd.read_csv(input[0]).power
+ feedin_timeseries = feedin_timeseries.div(feedin_timeseries.sum())
+ feedin_timeseries.to_csv(output[0])
diff --git a/digipipe/store/datasets/renewable_feedin/data/.gitkeep b/digipipe/store/datasets/renewable_feedin/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/renewable_feedin/dataset.md b/digipipe/store/datasets/renewable_feedin/dataset.md
new file mode 100644
index 00000000..2d525af5
--- /dev/null
+++ b/digipipe/store/datasets/renewable_feedin/dataset.md
@@ -0,0 +1,16 @@
+# EE-Einspeisezeitreihen
+
+Einspeisezeitreihen für Erneuerbare Energien. Als Wetterjahr wird 2011
+verwendet, siehe [Szenarien](../../../../docs/sections/scenarios.md).
+
+Raw dataset mit methodischer Beschreibung:
+[renewables.ninja_feedin](../../raw/renewables.ninja_feedin/dataset.md)
+
+## Einspeisezeitreihen
+
+Zeitreihe normiert auf Summe=1 für
+
+- Windenergie: `wind_feedin_timeseries.csv`
+- Photovoltaik: `pv_feedin_timeseries.csv`
+- Solarthermie: `st_feedin_timeseries.csv`
+- Laufwasserkraft: `ror_feedin_timeseries.csv`
diff --git a/digipipe/store/datasets/renewable_feedin/scripts/create.py b/digipipe/store/datasets/renewable_feedin/scripts/create.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/rli_pv_wfr_region/config.yml b/digipipe/store/datasets/rli_pv_wfr_region/config.yml
new file mode 100644
index 00000000..befbab8d
--- /dev/null
+++ b/digipipe/store/datasets/rli_pv_wfr_region/config.yml
@@ -0,0 +1,34 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Files to be clipped to region
+files_clip:
+ [
+ "air_traffic_control_system.gpkg",
+ "aviation.gpkg",
+ "biosphere_reserve.gpkg",
+ "drinking_water_protection_area.gpkg",
+ "fauna_flora_habitat.gpkg",
+ "floodplain.gpkg",
+ "forest.gpkg",
+ "grid.gpkg",
+ "industry.gpkg",
+ "landscape_protection_area.gpkg",
+ "less_favoured_areas_agricultural.gpkg",
+ "military.gpkg",
+ "national_park.gpkg",
+ "nature_conservation_area.gpkg",
+ "potentialarea_pv_agriculture_lfa-off.gpkg",
+ "potentialarea_pv_road_railway.gpkg",
+ "railway.gpkg",
+ "road.gpkg",
+ "road_railway-500m.gpkg",
+ "settlement-0m.gpkg",
+ "slope.gpkg",
+ "soil_quality_high.gpkg",
+ "soil_quality_low.gpkg",
+ "special_protection_area.gpkg",
+ "water.gpkg",
+ "wetland_ramsar.gpkg"
+ ]
diff --git a/digipipe/store/datasets/rli_pv_wfr_region/create.smk b/digipipe/store/datasets/rli_pv_wfr_region/create.smk
new file mode 100644
index 00000000..02b2589e
--- /dev/null
+++ b/digipipe/store/datasets/rli_pv_wfr_region/create.smk
@@ -0,0 +1,48 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import geopandas as gpd
+from digipipe.config import GLOBAL_CONFIG
+from digipipe.scripts.geo import (
+ overlay,
+ convert_to_multipolygon,
+ write_geofile
+)
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path(
+ "datasets", "rli_pv_wfr_region", data_dir=True)
+
+rule clip_to_region:
+ """
+ Clip to region and reproject
+ """
+ input:
+ geodata=get_abs_dataset_path(
+ "preprocessed", "rli_pv_wfr", data_dir=True
+ ) / "{file}.gpkg",
+ region=rules.datasets_bkg_vg250_region_create.output
+ output:
+ geodata=DATASET_PATH / "{file}_region.gpkg"
+ run:
+ print(f"Clipping layer {wildcards.file} to region...")
+
+ geodata = gpd.read_file(input.geodata).to_crs(
+ GLOBAL_CONFIG["global"]["geodata"]["crs"]
+ )
+ geodata = overlay(
+ gdf=geodata,
+ gdf_overlay=gpd.read_file(input.region[0]),
+ ).drop(columns=["id"])
+
+ if len(geodata) == 0:
+ print(" Layer has no data in region!")
+
+ write_geofile(
+ gdf=convert_to_multipolygon(geodata),
+ file=output.geodata,
+ layer_name=wildcards.file,
+ )
diff --git a/digipipe/store/datasets/rli_pv_wfr_region/data/.gitkeep b/digipipe/store/datasets/rli_pv_wfr_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/rli_pv_wfr_region/dataset.md b/digipipe/store/datasets/rli_pv_wfr_region/dataset.md
new file mode 100644
index 00000000..aa48988b
--- /dev/null
+++ b/digipipe/store/datasets/rli_pv_wfr_region/dataset.md
@@ -0,0 +1,8 @@
+# Geodaten PV- und Windflächenrechner
+
+Geodaten aus dem [PV- und Windflächenrechner](https://www.agora-energiewende.de/service/pv-und-windflaechenrechner/),
+extrahiert, zu LAEA Europe (EPSG:3035) umprojiziert und auf die Regionsgrenzen
+zugeschnitten.
+
+Preprocessed dataset:
+[rli_pv_windflaechenrechner](../../preprocessed/rli_pv_wfr/dataset.md)
diff --git a/digipipe/store/datasets/technology_data/config.yml b/digipipe/store/datasets/technology_data/config.yml
new file mode 100644
index 00000000..17d3af5a
--- /dev/null
+++ b/digipipe/store/datasets/technology_data/config.yml
@@ -0,0 +1,3 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
diff --git a/digipipe/store/datasets/technology_data/create.smk b/digipipe/store/datasets/technology_data/create.smk
new file mode 100644
index 00000000..fe0c27a9
--- /dev/null
+++ b/digipipe/store/datasets/technology_data/create.smk
@@ -0,0 +1,24 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path(
+ "datasets", "technology_data", data_dir=True)
+
+rule copy_files:
+ """
+ Copy technology data from raw
+ """
+ input:
+ get_abs_dataset_path(
+ "raw", "technology_data") / "data" / "technology_data.json"
+ output:
+ DATASET_PATH / "technology_data.json"
+ shell:
+ """
+ cp -p {input} {output}
+ """
diff --git a/digipipe/store/datasets/technology_data/data/.gitkeep b/digipipe/store/datasets/technology_data/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/datasets/technology_data/dataset.md b/digipipe/store/datasets/technology_data/dataset.md
new file mode 100644
index 00000000..18a2b7b9
--- /dev/null
+++ b/digipipe/store/datasets/technology_data/dataset.md
@@ -0,0 +1,5 @@
+# Technologiedaten
+
+Allgemeine Technologiedaten.
+
+Raw dataset: [technology_data](../../raw/technology_data/dataset.md)
diff --git a/digipipe/store/preprocessed/TEMPLATE/config.yml b/digipipe/store/preprocessed/TEMPLATE/config.yml
new file mode 100644
index 00000000..441e79a0
--- /dev/null
+++ b/digipipe/store/preprocessed/TEMPLATE/config.yml
@@ -0,0 +1,5 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Further custom configuration goes here
diff --git a/digipipe/store/preprocessed/TEMPLATE/create.smk b/digipipe/store/preprocessed/TEMPLATE/create.smk
new file mode 100644
index 00000000..52a00a07
--- /dev/null
+++ b/digipipe/store/preprocessed/TEMPLATE/create.smk
@@ -0,0 +1,24 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+Please add a docstring with a short description to each rule.
+"""
+from pathlib import Path
+from digipipe.store.utils import get_abs_dataset_path
+
+#configfile: get_abs_dataset_path("preprocessed", ".TEMPLATE", data_dir=False) / "config.yml"
+
+
+# Use function `get_abs_dataset_path()` to get path to dataset
+rule template_simple_copy_file1:
+ input: get_abs_dataset_path("raw", ".TEMPLATE") / "some_timeseries.csv"
+ output: get_abs_dataset_path("preprocessed", ".TEMPLATE") / "some_timeseries.csv"
+ shell: "cp -p {input} {output}"
+
+# Or you can use relative paths which may break as snakemake can be invoked
+# from directories digipipe/ or digipipe/workflow/.
+rule template_simple_copy_file2:
+ input: Path(".") / "store" / "raw" / ".TEMPLATE" / "data" / "some_timeseries.csv"
+ output: Path(".") / "store" / "preprocessed" / ".TEMPLATE" / "data" / "some_timeseries.csv"
+ shell: "cp -p {input} {output}"
diff --git a/digipipe/store/preprocessed/TEMPLATE/data/.gitkeep b/digipipe/store/preprocessed/TEMPLATE/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/TEMPLATE/dataset.md b/digipipe/store/preprocessed/TEMPLATE/dataset.md
new file mode 100644
index 00000000..15bd7af4
--- /dev/null
+++ b/digipipe/store/preprocessed/TEMPLATE/dataset.md
@@ -0,0 +1,4 @@
+# Name des Datensatzes
+
+Eine kurze Beschreibung des Datensatzes.
+Diese hilft der Dokumentation und bei der Erstellung des Berichts.
diff --git a/digipipe/store/preprocessed/ageb_energy_balance/__init__.py b/digipipe/store/preprocessed/ageb_energy_balance/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/ageb_energy_balance/config.yml b/digipipe/store/preprocessed/ageb_energy_balance/config.yml
new file mode 100644
index 00000000..03f9fb2d
--- /dev/null
+++ b/digipipe/store/preprocessed/ageb_energy_balance/config.yml
@@ -0,0 +1,54 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Params for PDF extraction
+
+# Relative area on page in %
+area: [58, 0, 86, 100]
+sectors:
+ hh:
+ page: 10
+ column_names:
+ ["carrier",
+ "space_heating",
+ "hot_water",
+ "process_heat",
+ "heat_total",
+ "space_cooling",
+ "process_cooling",
+ "cooling_total",
+ "mech_energy",
+ "ICT",
+ "lighting",
+ "total"]
+ cts:
+ page: 14
+ column_names:
+ ["carrier",
+ "space_heating",
+ "hot_water",
+ "process_heat",
+ "heat_total",
+ "space_cooling",
+ "process_cooling",
+ "cooling_total",
+ "mech_energy",
+ "ICT",
+ "lighting",
+ "total"]
+ ind:
+ page: 6
+ column_names:
+ ["carrier",
+ "space_heating",
+ "hot_water",
+ "process_heat",
+ "heat_total",
+ "space_cooling",
+ "process_cooling",
+ "cooling_total",
+ "mech_energy",
+ "ICT",
+ "lighting",
+ "total"]
diff --git a/digipipe/store/preprocessed/ageb_energy_balance/create.smk b/digipipe/store/preprocessed/ageb_energy_balance/create.smk
new file mode 100644
index 00000000..9d58a1be
--- /dev/null
+++ b/digipipe/store/preprocessed/ageb_energy_balance/create.smk
@@ -0,0 +1,42 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import tabula
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "ageb_energy_balance", data_dir=True)
+
+rule convert:
+ """
+ Convert PDF to CSVs
+ """
+ input:
+ get_abs_dataset_path("raw", "ageb_energy_balance")
+ / "data" / "AGEB_21p2_V3_20221222.pdf"
+ output:
+ DATASET_PATH / "ageb_energy_balance_germany_{sector}_twh_2021.csv",
+ run:
+ # Read data from PDF
+ demand = tabula.read_pdf(
+ input[0],
+ pages=[config["sectors"][wildcards.sector]["page"]],
+ area=config["area"],
+ relative_area=True
+ )[0]
+
+ # Processing
+ demand.drop(columns=["Unnamed: 0"], inplace=True)
+ demand.columns=config["sectors"][wildcards.sector]["column_names"]
+ demand = demand.set_index("carrier").replace({"-": 0})
+ for col in demand.columns:
+ demand[col] = demand[col].str.replace(
+ ',', '.').str.replace(" ", "").astype(float).fillna(0)
+ demand.drop("Insgesamt", axis=0, inplace=True)
+
+ # PJ to TWh
+ demand = demand.div(3.6)
+
+ # Dump as CSV
+ demand.to_csv(output[0])
diff --git a/digipipe/store/preprocessed/ageb_energy_balance/data/.gitkeep b/digipipe/store/preprocessed/ageb_energy_balance/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/ageb_energy_balance/dataset.md b/digipipe/store/preprocessed/ageb_energy_balance/dataset.md
new file mode 100644
index 00000000..27d2a7f3
--- /dev/null
+++ b/digipipe/store/preprocessed/ageb_energy_balance/dataset.md
@@ -0,0 +1,7 @@
+# AGEB – Anwendungsbilanzen für die Endenergiesektoren 2011 bis 2021
+
+Detaillierte Anwendungsbilanzen der Endenergiesektoren für 2020 und 2021 sowie
+zusammenfassende Zeitreihen zum Endenergieverbrauch nach Energieträgern und
+Anwendungszwecken für Jahre von 2011 bis 2021 der AG Energiebilanzen.
+
+Aus PDF extrahierte Tabellenwerte für Haushalte, GHD und Industrie.
diff --git a/digipipe/store/preprocessed/ba_employment/__init__.py b/digipipe/store/preprocessed/ba_employment/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/ba_employment/config.yml b/digipipe/store/preprocessed/ba_employment/config.yml
new file mode 100644
index 00000000..3ccceb9e
--- /dev/null
+++ b/digipipe/store/preprocessed/ba_employment/config.yml
@@ -0,0 +1,21 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Excel file
+excel_file:
+ sheet_name: "Gemeindedaten"
+ usecols: "A, K, O"
+ skiprows: 8
+ names:
+ [
+ "ags",
+ "employees_total",
+ "companies_total"
+ ]
+ dtype:
+ {
+ "ags": str,
+ "employees_total": str,
+ "companies_total": str
+ }
diff --git a/digipipe/store/preprocessed/ba_employment/create.smk b/digipipe/store/preprocessed/ba_employment/create.smk
new file mode 100644
index 00000000..1e34e675
--- /dev/null
+++ b/digipipe/store/preprocessed/ba_employment/create.smk
@@ -0,0 +1,35 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "ba_employment")
+
+rule unzip:
+ """
+ Unzip file
+ """
+ input:
+ get_abs_dataset_path("raw", "ba_employment") / "data" / "gemband-dlk-0-202206-zip.zip"
+ output:
+ files = DATASET_PATH / "data" / "gemband_dlk_0.xlsb"
+ params:
+ outpath=DATASET_PATH / "data"
+ shell:
+ """
+ unzip -j {input} -d {params.outpath}
+ """
+
+rule create:
+ """
+ Extract municipality data from Excel file and save to CSV
+ """
+ input:
+ DATASET_PATH / "data" / "gemband_dlk_0.xlsb"
+ output:
+ DATASET_PATH / "data" / "employment_muns.csv"
+ script:
+ DATASET_PATH / "scripts" / "create.py"
diff --git a/digipipe/store/preprocessed/ba_employment/data/.gitkeep b/digipipe/store/preprocessed/ba_employment/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/ba_employment/dataset.md b/digipipe/store/preprocessed/ba_employment/dataset.md
new file mode 100644
index 00000000..a4ea6297
--- /dev/null
+++ b/digipipe/store/preprocessed/ba_employment/dataset.md
@@ -0,0 +1,6 @@
+# Sozialversicherungspflichtig Beschäftigte und Betriebe
+
+Gemeindedaten der sozialversicherungspflichtig Beschäftigten am 30.06.2022 nach
+Wohn- und Arbeitsort - Deutschland, Länder, Kreise und Gemeinden (Jahreszahlen)
+der Bundesagentur für Arbeit.
+Anzahl Beschäftigte und Betriebe extrahiert und in CSV konvertiert.
diff --git a/digipipe/store/preprocessed/ba_employment/scripts/create.py b/digipipe/store/preprocessed/ba_employment/scripts/create.py
new file mode 100644
index 00000000..efe3ef85
--- /dev/null
+++ b/digipipe/store/preprocessed/ba_employment/scripts/create.py
@@ -0,0 +1,27 @@
+import pandas as pd
+
+
+def process() -> None:
+ # Get Excel config
+ excel_cfg = snakemake.config["excel_file"]
+
+ # Read file
+ data = pd.read_excel(
+ snakemake.input[0],
+ **excel_cfg,
+ engine="pyxlsb",
+ )
+
+ # Set empty values to 0 and convert dtypes
+ data = data.replace(["*", " "], 0)
+ data = data.astype({"employees_total": int, "companies_total": int})
+
+ # Drop empoty and aggregated rows
+ data = data.loc[data.ags.str.len() == 8].set_index("ags")
+
+ # Write
+ data.to_csv(snakemake.output[0])
+
+
+if __name__ == "__main__":
+ process()
diff --git a/digipipe/store/preprocessed/bkg_vg250/__init__.py b/digipipe/store/preprocessed/bkg_vg250/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/bkg_vg250/config.yml b/digipipe/store/preprocessed/bkg_vg250/config.yml
new file mode 100644
index 00000000..fd895984
--- /dev/null
+++ b/digipipe/store/preprocessed/bkg_vg250/config.yml
@@ -0,0 +1,14 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Layers to be extracted
+layers:
+ [
+ "vg250_gem",
+ "vg250_krs",
+ "vg250_lan",
+ "vg250_rbz",
+ "vg250_vwg",
+ "vg250_sta"
+ ]
diff --git a/digipipe/store/preprocessed/bkg_vg250/create.smk b/digipipe/store/preprocessed/bkg_vg250/create.smk
new file mode 100644
index 00000000..105d0838
--- /dev/null
+++ b/digipipe/store/preprocessed/bkg_vg250/create.smk
@@ -0,0 +1,26 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "bkg_vg250")
+
+rule create:
+ input:
+ get_abs_dataset_path("raw", "bkg_vg250") / "data" / "vg250_01-01.utm32s.gpkg.ebenen.zip"
+ output:
+ DATASET_PATH / "data" / "bkg_vg250.gpkg"
+ params:
+ outpath=DATASET_PATH / "data",
+ original_file=DATASET_PATH / "data" / "DE_VG250.gpkg",
+ file_path_in_zip=str("vg250_01-01.utm32s.gpkg.ebenen/vg250_ebenen_0101/DE_VG250.gpkg"),
+ layers=" ".join(config["layers"])
+ shell:
+ """
+ unzip -j {input} {params.file_path_in_zip} -d {params.outpath}
+ ogr2ogr -f GPKG -t_srs EPSG:3035 {output} {params.original_file} {params.layers}
+ rm {params.original_file}
+ """
diff --git a/digipipe/store/preprocessed/bkg_vg250/data/.gitkeep b/digipipe/store/preprocessed/bkg_vg250/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/bkg_vg250/dataset.md b/digipipe/store/preprocessed/bkg_vg250/dataset.md
new file mode 100644
index 00000000..a0c0e25a
--- /dev/null
+++ b/digipipe/store/preprocessed/bkg_vg250/dataset.md
@@ -0,0 +1,4 @@
+# Administative areas of Germany
+
+Geodata of administrative areas (Verwaltungsgebiete 1:250 000) extracted,
+reprojected to LAEA Europe(EPSG:3035) and converted to Geopackage.
diff --git a/digipipe/store/preprocessed/bmwk_long_term_scenarios/config.yml b/digipipe/store/preprocessed/bmwk_long_term_scenarios/config.yml
new file mode 100644
index 00000000..b1e57502
--- /dev/null
+++ b/digipipe/store/preprocessed/bmwk_long_term_scenarios/config.yml
@@ -0,0 +1,149 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Files to be extracted from raw dataset
+files_extract:
+ [
+ "T45-Strom_buildings_heating_demand_by_carrier.csv",
+ "T45-Strom_buildings_heating_structure_by_technology.csv",
+ "T45-Strom_cts_demand.csv",
+ "T45-Strom_electricity_installed_power.csv",
+ "T45-Strom_Generation_Heatgrids_Germany.csv",
+ "T45-Strom_hh_demand.csv",
+ "T45-Strom_ind_demand.csv",
+ "TN-Strom_buildings_heating_demand_by_carrier.csv",
+ "TN-Strom_buildings_heating_structure_by_technology.csv",
+ "TN-Strom_cts_demand.csv",
+ "TN-Strom_hh_demand.csv",
+ "TN-Strom_ind_demand.csv",
+ ]
+
+# New column names
+rename_columns:
+ T45-Strom_buildings_heating_demand_by_carrier:
+ {" Jahr / Year": "year",
+ "Energiebedarf in TWh / Energy Demand in TWh": "demand",
+ "Energieträger / Energy Carrier": "carrier"}
+ T45-Strom_buildings_heating_structure_by_technology:
+ {"Jahr / Year": "year",
+ "Anzahl der Heizungen in Mio. / Number of heaters in mil.": "heating_units_count",
+ "Typ / type": "heating_technology"}
+ T45-Strom_cts_demand:
+ {"Jahr / Year": "year",
+ "Energiebedarf in TWh / Energy Demand in TWh": "demand",
+ "Energieträger / Energy Carrier": "carrier"}
+ T45-Strom_electricity_installed_power:
+ {"Jahr / Year": "year",
+ "Technologie / Technology": "technology",
+ "Leistung in GW / Capacity in GW": "capacity"}
+ T45-Strom_Generation_Heatgrids_Germany:
+ {"Jahr / Year": "year",
+ "Technologie / Technology": "technology",
+ "Erzeugung in TWh / Generation in TWh": "generation"}
+ T45-Strom_hh_demand:
+ {"Jahr / Year": "year",
+ "Energiebedarf in TWh / Energy Demand in TWh": "demand",
+ "Energieträger / Energy Carrier": "carrier"}
+ T45-Strom_ind_demand:
+ {"Jahr / Year": "year",
+ "Energiebedarf in TWh / Energy Demand in TWh": "demand",
+ "Energieträger / Energy Carrier": "carrier"}
+ TN-Strom_buildings_heating_demand_by_carrier:
+ {"Jahr": "year",
+ "Energiebedarf in TWh": "demand",
+ "Typ": "carrier"}
+ TN-Strom_buildings_heating_structure_by_technology:
+ {"Jahr": "year",
+ "Anlagen in Mio.": "heating_units_count",
+ "Typ": "heating_technology"}
+ TN-Strom_cts_demand:
+ {"Jahr": "year",
+ "TWh": "demand",
+ "Typ": "carrier"}
+ TN-Strom_hh_demand:
+ {"Jahr": "year",
+ "TWh": "demand",
+ "Typ": "carrier"}
+ TN-Strom_ind_demand:
+ {"Jahr": "year",
+ "TWh": "demand",
+ "Energy Carrier": "carrier"}
+
+# New names for carriers and technologies
+rename_entries:
+ {'Abfall KWK': 'waste_chp',
+ 'Andere': 'other',
+ 'Andere Fossile': 'other_fossil_fuel',
+ 'Andere fossile': 'other_fossil_fuel',
+ 'Andere fossile KWK': 'other_fossil_fuel_chp',
+ 'BHKW': 'chp_small',
+ 'Biobenzin': 'biopetrol',
+ 'Biodiesel': 'biodiesel',
+ 'Biogas': 'biogas',
+ 'Biokraftstoffe': 'biofuels',
+ 'Biomasse': 'biomass',
+ 'Biomasse, fest': 'biomass',
+ 'Biomasse KWK': 'biomass_chp',
+ 'Biotreibstoffe': 'biofuels',
+ 'Braunkohle': 'lignite',
+ 'EE-Methan': 'renewable_methane',
+ 'Elektrokessel': 'electricity_direct_heating',
+ 'Erdgas': 'natural_gas',
+ 'Fernwärme': 'district_heating',
+ 'Fossile': 'fossil_fuel',
+ 'Gas': 'natural_gas',
+ 'Gas-Heizkessel': 'gas_boiler',
+ 'Gas Heizkessel': 'gas_boiler',
+ 'Gas KWK': 'methane_chp',
+ 'Geothermie': 'geothermal',
+ 'Gichtgas': 'blast_furnace_gas',
+ 'Großwärmepumpen': 'heat_pump',
+ 'Hackschnitzel': 'woodchip',
+ 'Hackschnitzel-Heizkessel': 'woodchip_boiler',
+ 'Hackschnitzel-Heizungen': 'woodchip_boiler',
+ 'Heizöl': 'fuel_oil',
+ 'Hybrid-Wärmepumpen': 'hybrid_heat_pump',
+ 'Kohle': 'lignite',
+ 'Kokereigas, Stadtgas, LPG, Raffineriegas': 'coke_illuminating_refinery_gas_lpg',
+ 'Koks': 'coke',
+ 'Methan': 'methane',
+ 'Mineralöle': 'mineral_oil',
+ 'Müll nicht erneuerbar': 'waste_non-renewable',
+ 'Müll, erneuerbar': 'waste_renewable',
+ 'Müll, nicht erneuerbar': 'waste_non-renewable',
+ 'Nah- und Fernwärme': 'district_heating',
+ 'PV': 'pv',
+ 'Pellet-Heizkessel': 'pellet_boiler',
+ 'Pellets': 'pellet',
+ 'Pflanzenöl': 'plant_oil',
+ 'PtG': 'ptg',
+ 'Solarenergie': 'solar_energy',
+ 'Solarthermie': 'solar_thermal',
+ 'Solarthermie Hzg. u. TW': 'solar_thermal',
+ 'Steinkohle': 'black_coal',
+ 'Strom': 'electricity',
+ 'Strom Hilfsenergie': 'electricity_auxiliary',
+ 'Strom Wärmepumpe': 'electricity_heat_pump',
+ 'Strom direkt': 'electricity_direct_heating',
+ 'Strom direktelektrisch': 'electricity_direct_heating',
+ 'Strom-Direktheizungen': 'electric_boiler',
+ 'Umgebungswärme': 'ambient_heat',
+ 'Umgebungswärme, Boden': 'ambient_heat_soil',
+ 'Umgebungswärme, Luft': 'ambient_heat_air',
+ 'Umweltwärme Wärmepumpe': 'ambient_heat_heat_pump',
+ 'WP-Strom': 'electricity_heat_pump',
+ 'WP-Umgebungswärme': 'ambient_heat_heat_pump',
+ 'Wasserkraft': 'hydro',
+ 'Wasserstoff': 'hydrogen',
+ 'Wasserstoff Kessel': 'hydrogen_boiler',
+ 'Wasserstoff KWK': 'hydrogen_chp',
+ 'Wind an Land': 'wind_onshore',
+ 'Wind auf See': 'sind_offshore',
+ 'Wärmenetzanschlüsse': 'district_heating_connection',
+ 'Wärmenetze': 'district_heating_connection',
+ 'Wärmepumpen': 'heat_pump',
+ 'feste biogene Stoffe': 'biomass',
+ 'Öl-Heizkessel': 'fuel_oil_boiler',
+ 'Übrige Erneuerbare': 'other_renewables',
+ 'Übrige, erneuerbar': 'other_renewables'}
diff --git a/digipipe/store/preprocessed/bmwk_long_term_scenarios/create.smk b/digipipe/store/preprocessed/bmwk_long_term_scenarios/create.smk
new file mode 100644
index 00000000..11317abc
--- /dev/null
+++ b/digipipe/store/preprocessed/bmwk_long_term_scenarios/create.smk
@@ -0,0 +1,74 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import json
+import os
+import pandas as pd
+from pathlib import Path
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "bmwk_long_term_scenarios", data_dir=True)
+
+rule create:
+ """
+ Extract files
+ """
+ input:
+ get_abs_dataset_path(
+ "raw", "bmwk_long_term_scenarios") / "data" /
+ "bmwk_long_term_scenarios.zip"
+ output:
+ files=[DATASET_PATH / f for f in config["files_extract"]]
+ params:
+ outpath=DATASET_PATH,
+ files_extract=" ".join(config["files_extract"])
+ shell:
+ """
+ unzip -j {input} {params.files_extract} -d {params.outpath}
+ """
+
+rule rename_columns_carriers:
+ """
+ Rename columns, carriers and technologies
+ """
+ input: DATASET_PATH / "{file}.csv"
+ output: DATASET_PATH / "{file}_reformatted.csv"
+ run:
+ data = pd.read_csv(input[0])
+
+ # Rename columns
+ data.rename(
+ columns=config["rename_columns"].get(wildcards.file),
+ inplace=True
+ )
+ if not all(
+ c in config["rename_columns"].get(wildcards.file).values()
+ for c in data.columns
+ ):
+ raise ValueError("At least one column was not renamed!")
+ # Rename carriers and technologies
+ data = data.replace(config["rename_entries"])
+
+ data.to_csv(output[0], index=False)
+ #os.remove(input[0])
+
+rule create_captions:
+ """
+ Create attribute captions for app
+ """
+ input: [DATASET_PATH / f for f in config["files_extract"]]
+ output: DATASET_PATH / "bmwk_long_term_scenarios_attribute_captions.json"
+ run:
+ captions = dict(
+ datasets_caption_map={
+ Path(f).stem: "bmwk_long_term_scenarios" for f in input},
+ captions={"bmwk_long_term_scenarios": {
+ v: k for k, v in config["rename_entries"].items()}
+ },
+ )
+ with open(output[0], "w", encoding="utf8") as f:
+ json.dump(captions, f, indent=4)
diff --git a/digipipe/store/preprocessed/bmwk_long_term_scenarios/data/.gitkeep b/digipipe/store/preprocessed/bmwk_long_term_scenarios/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md b/digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md
new file mode 100644
index 00000000..45c723b4
--- /dev/null
+++ b/digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md
@@ -0,0 +1,7 @@
+# BMWK Langfristszenarien
+
+Langfristszenarien des Bundesministerium für Wirtschaft und Klimaschutz, Daten
+auf Landesebene, extrahiert.
+
+Raw dataset:
+[bmwk_long_term_scenarios](../../raw/bmwk_long_term_scenarios/dataset.md)
diff --git a/digipipe/store/preprocessed/bnetza_mastr/__init__.py b/digipipe/store/preprocessed/bnetza_mastr/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/bnetza_mastr/config.yml b/digipipe/store/preprocessed/bnetza_mastr/config.yml
new file mode 100644
index 00000000..3a350ad0
--- /dev/null
+++ b/digipipe/store/preprocessed/bnetza_mastr/config.yml
@@ -0,0 +1,22 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Files to be extracted from raw dataset
+files_extract:
+ [
+ "bnetza_mastr_biomass_raw.csv",
+ "bnetza_mastr_combustion_raw.csv",
+ "bnetza_mastr_electricity_consumer_raw.csv",
+ "bnetza_mastr_gas_consumer_raw.csv",
+ "bnetza_mastr_gas_producer_raw.csv",
+ "bnetza_mastr_gas_storage_extended_raw.csv",
+ "bnetza_mastr_grid_connections_raw.csv",
+ "bnetza_mastr_gsgk_raw.csv",
+ "bnetza_mastr_hydro_raw.csv",
+ "bnetza_mastr_locations_extended_raw.csv",
+ "bnetza_mastr_solar_raw.csv",
+ "bnetza_mastr_storage_raw.csv",
+ "bnetza_mastr_storage_unit_raw.csv",
+ "bnetza_mastr_wind_raw.csv",
+ ]
diff --git a/digipipe/store/preprocessed/bnetza_mastr/create.smk b/digipipe/store/preprocessed/bnetza_mastr/create.smk
new file mode 100644
index 00000000..5b67dad6
--- /dev/null
+++ b/digipipe/store/preprocessed/bnetza_mastr/create.smk
@@ -0,0 +1,22 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "bnetza_mastr")
+
+rule create:
+ input:
+ get_abs_dataset_path("raw", "bnetza_mastr") / "data" / "bnetza_open_mastr_2022-12-19.zip"
+ output:
+ files=[DATASET_PATH / "data" / f for f in config["files_extract"]]
+ params:
+ outpath=DATASET_PATH / "data",
+ files_extract=" ".join([f"bnetza_open_mastr_2022-12-19/{f}" for f in config["files_extract"]])
+ shell:
+ """
+ unzip -j {input} {params.files_extract} -d {params.outpath}
+ """
diff --git a/digipipe/store/preprocessed/bnetza_mastr/data/.gitkeep b/digipipe/store/preprocessed/bnetza_mastr/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/bnetza_mastr/dataset.md b/digipipe/store/preprocessed/bnetza_mastr/dataset.md
new file mode 100644
index 00000000..d36d7e0e
--- /dev/null
+++ b/digipipe/store/preprocessed/bnetza_mastr/dataset.md
@@ -0,0 +1,3 @@
+# Erzeugungsanlagen aus Marktstammdatenregister
+
+Erzeugungsanlagen aus dem MaStR für ausgewählte Technologien.
diff --git a/digipipe/store/preprocessed/dbfz_biomass_capacity_rel/config.yml b/digipipe/store/preprocessed/dbfz_biomass_capacity_rel/config.yml
new file mode 100644
index 00000000..d8db3d15
--- /dev/null
+++ b/digipipe/store/preprocessed/dbfz_biomass_capacity_rel/config.yml
@@ -0,0 +1,25 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Years specified for the projections in the source
+years :
+ [
+ "2020",
+ "2050"
+ ]
+
+# Type of supply: Decentralized and centralized heat supply
+supplies :
+ [
+ "central",
+ "decentral"
+ ]
+
+# For simplicity, the projections for 2050 are assigned to 2045 and those for
+# 2020 to 2022.
+year_mapping :
+ {
+ "2050": "2045",
+ "2020": "2022"
+ }
diff --git a/digipipe/store/preprocessed/dbfz_biomass_capacity_rel/create.smk b/digipipe/store/preprocessed/dbfz_biomass_capacity_rel/create.smk
new file mode 100644
index 00000000..ce54176b
--- /dev/null
+++ b/digipipe/store/preprocessed/dbfz_biomass_capacity_rel/create.smk
@@ -0,0 +1,82 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import pandas as pd
+from digipipe.store.utils import get_abs_dataset_path
+
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "dbfz_biomass_capacity_rel")
+
+rule create:
+ input:
+ get_abs_dataset_path(
+ "raw", "dbfz_biomass_heat_capacities"
+ ) / "data" / "dbfz_biomass_heat_capacities.csv"
+ output:
+ output_cen=DATASET_PATH / "data" / "dbfz_biomass_capacity_rel_central.csv",
+ output_dec=DATASET_PATH/ "data" / "dbfz_biomass_capacity_rel_decentral.csv"
+ run:
+ for supply in config["supplies"]:
+ all_shares_biomass_cap = pd.DataFrame()
+ for year in config["years"]:
+ # Read raw file
+ biomass_cap = pd.read_csv(
+ input[0],
+ sep=";",
+ usecols=["carrier", "tech", "capacity_[MW]_" + year, supply],
+ )
+
+ # Drop nan values
+ biomass_cap = biomass_cap.dropna()
+
+ # Group by the columns "carrier" and "tech"
+ biomass_cap = biomass_cap.groupby(by=["carrier", "tech"]).sum(
+ numeric_only=True
+ )
+
+ # Calculate the share of the installed capacity per grouped
+ # technologies
+ share_biomass_cap = biomass_cap.apply(
+ lambda biomass_cap: biomass_cap / biomass_cap.sum()
+ )
+ # Rename the column and drop the year from the name of the column with
+ # the capacity
+ share_biomass_cap.rename(
+ {"capacity_[MW]_" + year: "capacity_rel"}, axis=1, inplace=True
+ )
+
+ # Drop multiindex made by grouping
+ share_biomass_cap.reset_index(inplace=True)
+
+ # Write tech into carrier col with the convention:
+ # "carrier_tech"
+ share_biomass_cap["carrier"] = share_biomass_cap[
+ ["carrier", "tech"]
+ ].agg("_".join, axis=1)
+
+ # Delete redundant column "tech"
+ del share_biomass_cap["tech"]
+
+ # Add year according to year mapping
+ share_biomass_cap["year"] = config["year_mapping"][year]
+
+ # Set index on the year to move the column to the front
+ share_biomass_cap.set_index("year", inplace=True)
+ share_biomass_cap.reset_index(inplace=True)
+
+ # Update final df with all shares
+ all_shares_biomass_cap = pd.concat(
+ [all_shares_biomass_cap, share_biomass_cap]
+ )
+
+ # Save the file
+ if supply == "central":
+ all_shares_biomass_cap.to_csv(output.output_cen, index=False)
+ elif supply == "decentral":
+ all_shares_biomass_cap.to_csv(output.output_dec, index=False)
+ else:
+ raise ValueError("Please provide either 'central' or 'decentral'"
+ "with 'supply'")
diff --git a/digipipe/store/preprocessed/dbfz_biomass_capacity_rel/data/.gitkeep b/digipipe/store/preprocessed/dbfz_biomass_capacity_rel/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/dbfz_biomass_capacity_rel/dataset.md b/digipipe/store/preprocessed/dbfz_biomass_capacity_rel/dataset.md
new file mode 100644
index 00000000..b186f864
--- /dev/null
+++ b/digipipe/store/preprocessed/dbfz_biomass_capacity_rel/dataset.md
@@ -0,0 +1,15 @@
+# Anteile von Biomasse-Konversionsanlagen anhand installierter Leistung
+
+Berechnung der Anteile der installierten Leistung an der gesamten installierten
+Leistung der Biomasse-Konversionsanlagen.
+
+Die installierten Leistungen werden
+[dbfz_biomass_heat_capacities](../../raw/dbfz_biomass_heat_capacities/dataset.md)
+entnommen. Sie werden nach Energieträger (Biogas, Methan oder Holz) und
+Technologie (BHKW (bpchp), Turbine mit Kondensationsentnahme (extchp) oder
+Ofen (oven)) zusammengefasst. Anschließend wird der Anteil der installierten
+Leistung an der gesamten installierten Leistung der Biomasse-Konversionsanlagen
+berechnet. Der Einfachheit halber werden die Projektionen für 2050 dem Jahr
+2045 und die für 2020 dem Jahr 2022 zugeordnet. Der Energieträger und die
+Technologie (vgl. [dbfz_biomass_heat_capacities](../../raw/dbfz_biomass_heat_capacities/dataset.md))
+werden in einer Spalte zusammengefasst.
diff --git a/digipipe/store/preprocessed/demandregio/__init__.py b/digipipe/store/preprocessed/demandregio/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/demandregio/config.yml b/digipipe/store/preprocessed/demandregio/config.yml
new file mode 100644
index 00000000..a5ad93e2
--- /dev/null
+++ b/digipipe/store/preprocessed/demandregio/config.yml
@@ -0,0 +1,22 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Files to be extracted from raw dataset
+files_extract:
+ [
+ "dr_cts_gas_timeseries_2011.csv",
+ "dr_cts_power_demand_2022.csv",
+ "dr_cts_power_timeseries_2022.csv",
+ "dr_hh_gas_timeseries_2011.csv",
+ "dr_hh_households_2011.csv",
+ "dr_hh_population.csv",
+ "dr_hh_power_demand_2022.csv",
+ "dr_hh_power_timeseries_2022.csv",
+ "dr_ind_gas_demand_2022.csv",
+ "dr_ind_gas_timeseries_2011.csv",
+ "dr_ind_gas_timeseries_2022.csv",
+ "dr_ind_power_demand_2022.csv",
+ "dr_ind_power_timeseries_2022.csv",
+ "dr_temperature_ambient_2011.csv",
+ ]
diff --git a/digipipe/store/preprocessed/demandregio/create.smk b/digipipe/store/preprocessed/demandregio/create.smk
new file mode 100644
index 00000000..70552080
--- /dev/null
+++ b/digipipe/store/preprocessed/demandregio/create.smk
@@ -0,0 +1,22 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "demandregio", data_dir=True)
+
+rule create:
+ input:
+ get_abs_dataset_path("raw", "demandregio") / "data" / "demandregio.zip"
+ output:
+ files=[DATASET_PATH / f for f in config["files_extract"]]
+ params:
+ outpath=DATASET_PATH,
+ files_extract=" ".join(config["files_extract"])
+ shell:
+ """
+ unzip -j {input} {params.files_extract} -d {params.outpath}
+ """
diff --git a/digipipe/store/preprocessed/demandregio/data/.gitkeep b/digipipe/store/preprocessed/demandregio/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/demandregio/dataset.md b/digipipe/store/preprocessed/demandregio/dataset.md
new file mode 100644
index 00000000..5648b787
--- /dev/null
+++ b/digipipe/store/preprocessed/demandregio/dataset.md
@@ -0,0 +1,7 @@
+# DemandRegio
+
+Regionalisierte Bevölkerungsprognose sowie Strom-, Wärme und Gasbedarf auf
+Landkreisebene, extrahiert.
+
+Enthält Jahresverbräuche und Zeitreihen für die Sektoren Haushalte, Gewerbe,
+Handel, Dienstleistungen (GHD) und Industrie für mehrere Zieljahre.
diff --git a/digipipe/store/preprocessed/destatis_gv/__init__.py b/digipipe/store/preprocessed/destatis_gv/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/destatis_gv/config.yml b/digipipe/store/preprocessed/destatis_gv/config.yml
new file mode 100644
index 00000000..380775cf
--- /dev/null
+++ b/digipipe/store/preprocessed/destatis_gv/config.yml
@@ -0,0 +1,31 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Excel file
+excel_file:
+ sheet_name:
+ {
+ "2010": "Gemeindedaten",
+ "2015": "Onlineprodukt_Gemeinden_311215",
+ "2020": "Onlineprodukt_Gemeinden",
+ "2021": "Onlineprodukt_Gemeinden",
+ "2022": "Onlineprodukt_Gemeinden"
+ }
+ usecols: "C:E, G, J"
+ skiprows: 6
+ names:
+ [
+ "Land",
+ "RB",
+ "Kreis",
+ "Gem",
+ "population"
+ ]
+ dtype:
+ {
+ "Land": str,
+ "RB": str,
+ "Kreis": str,
+ "Gem": str
+ }
diff --git a/digipipe/store/preprocessed/destatis_gv/create.smk b/digipipe/store/preprocessed/destatis_gv/create.smk
new file mode 100644
index 00000000..516e8bc2
--- /dev/null
+++ b/digipipe/store/preprocessed/destatis_gv/create.smk
@@ -0,0 +1,20 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "destatis_gv")
+
+rule create:
+ """
+ Extract municipality population data from Excel files and save to CSVs
+ """
+ input:
+ get_abs_dataset_path("raw", "destatis_gv") / "data" / "3112{year}_Auszug_GV.xlsx"
+ output:
+ DATASET_PATH / "data" / "3112{year}_Auszug_GV.csv"
+ script:
+ DATASET_PATH / "scripts" / "create.py"
diff --git a/digipipe/store/preprocessed/destatis_gv/data/.gitkeep b/digipipe/store/preprocessed/destatis_gv/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/destatis_gv/dataset.md b/digipipe/store/preprocessed/destatis_gv/dataset.md
new file mode 100644
index 00000000..6c0d0ce1
--- /dev/null
+++ b/digipipe/store/preprocessed/destatis_gv/dataset.md
@@ -0,0 +1,4 @@
+# Bevölkerung
+
+Einwohnerzahl nach Gemeinden des Statistischen Bundesamts für die Jahre
+2010, 2015, 2020, 2021, 2022.
diff --git a/digipipe/store/preprocessed/destatis_gv/scripts/create.py b/digipipe/store/preprocessed/destatis_gv/scripts/create.py
new file mode 100644
index 00000000..3e5083c3
--- /dev/null
+++ b/digipipe/store/preprocessed/destatis_gv/scripts/create.py
@@ -0,0 +1,30 @@
+import pandas as pd
+
+
+def process() -> None:
+ # Get Excel config and set sheet name
+ excel_cfg = snakemake.config["excel_file"]
+ excel_cfg["sheet_name"] = excel_cfg["sheet_name"][snakemake.wildcards.year]
+
+ data = pd.read_excel(
+ snakemake.input[0],
+ **excel_cfg,
+ engine="openpyxl",
+ )
+
+ # Drop non-municipal data
+ data = data.loc[~data.isnull().any(axis=1)]
+ data = data.assign(population=data.population.astype(int)).rename(
+ columns={"population": snakemake.wildcards.year}
+ )
+
+ # Create AGS and drop old cols
+ data = data.assign(ags=data.Land + data.RB + data.Kreis + data.Gem)
+
+ data[["ags", snakemake.wildcards.year]].to_csv(
+ snakemake.output[0], index=None
+ )
+
+
+if __name__ == "__main__":
+ process()
diff --git a/digipipe/store/preprocessed/dwd_temperature/config.yml b/digipipe/store/preprocessed/dwd_temperature/config.yml
new file mode 100644
index 00000000..17d3af5a
--- /dev/null
+++ b/digipipe/store/preprocessed/dwd_temperature/config.yml
@@ -0,0 +1,3 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
diff --git a/digipipe/store/preprocessed/dwd_temperature/create.smk b/digipipe/store/preprocessed/dwd_temperature/create.smk
new file mode 100644
index 00000000..ff15223d
--- /dev/null
+++ b/digipipe/store/preprocessed/dwd_temperature/create.smk
@@ -0,0 +1,32 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import pandas as pd
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "dwd_temperature", data_dir=True)
+
+rule create:
+ """
+ Calc mean temperatures for entire region
+ """
+ input:
+ temperature=get_abs_dataset_path(
+ "raw", "dwd_temperature") / "data" / "temperature_2011.csv"
+ output:
+ temperature=DATASET_PATH / "temperature_2011.csv"
+ run:
+ temp = pd.read_csv(
+ input.temperature,
+ index_col=["timestamp", "ags_id"],
+ )
+ # Calc regional mean from municipal values
+ temp = temp.astype("float").reset_index().drop(
+ columns=["ags_id"]).groupby("timestamp").agg("mean").round(2)
+ # Remove timestamp
+ temp = temp.reset_index().drop(columns=["timestamp"])
+ # Dump
+ temp.to_csv(output.temperature)
diff --git a/digipipe/store/preprocessed/dwd_temperature/data/.gitkeep b/digipipe/store/preprocessed/dwd_temperature/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/dwd_temperature/dataset.md b/digipipe/store/preprocessed/dwd_temperature/dataset.md
new file mode 100644
index 00000000..34fa574b
--- /dev/null
+++ b/digipipe/store/preprocessed/dwd_temperature/dataset.md
@@ -0,0 +1,6 @@
+# Temperatur
+
+Stündliche Mittelwerte der Luft- und Erdbodentemperatur für die Region ABW,
+Mittelwert für alle Gemeinden.
+
+Verwendet: [dwd_temperature](../../raw/dwd_temperature/dataset.md)
diff --git a/digipipe/store/preprocessed/eurostat_lau/__init__.py b/digipipe/store/preprocessed/eurostat_lau/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/eurostat_lau/config.yml b/digipipe/store/preprocessed/eurostat_lau/config.yml
new file mode 100644
index 00000000..d310985d
--- /dev/null
+++ b/digipipe/store/preprocessed/eurostat_lau/config.yml
@@ -0,0 +1,15 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Excel file
+excel_file:
+ sheet_name: "DE"
+ usecols: "A:C"
+ dtype:
+ {"LAU CODE": str}
+
+column_names:
+ ["nuts_code",
+ "lau_code",
+ "name"]
diff --git a/digipipe/store/preprocessed/eurostat_lau/create.smk b/digipipe/store/preprocessed/eurostat_lau/create.smk
new file mode 100644
index 00000000..2cb6b279
--- /dev/null
+++ b/digipipe/store/preprocessed/eurostat_lau/create.smk
@@ -0,0 +1,17 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "eurostat_lau")
+
+rule create:
+ """
+ Extract Germany's LAUs from Excel files and save to CSV
+ """
+ input: get_abs_dataset_path("raw", "eurostat_lau") / "data" / "EU-27-LAU-2022-NUTS-2021.xlsx"
+ output: DATASET_PATH / "data" / "germany_lau_codes.csv"
+ script: DATASET_PATH / "scripts" / "create.py"
diff --git a/digipipe/store/preprocessed/eurostat_lau/data/.gitkeep b/digipipe/store/preprocessed/eurostat_lau/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/eurostat_lau/dataset.md b/digipipe/store/preprocessed/eurostat_lau/dataset.md
new file mode 100644
index 00000000..0ce58069
--- /dev/null
+++ b/digipipe/store/preprocessed/eurostat_lau/dataset.md
@@ -0,0 +1,7 @@
+# Lokale Verwaltungseinheiten
+
+Lokale Verwaltungseinheiten (LAUs) von Eurostat, mit NUTS kompatibel. Diese
+LAUs sind die Bausteine der NUTS und umfassen die Gemeinden und Kommunen der
+Europäischen Union.
+
+Daten aus Excel extrahiert und in CSV exportiert.
diff --git a/digipipe/store/preprocessed/eurostat_lau/scripts/create.py b/digipipe/store/preprocessed/eurostat_lau/scripts/create.py
new file mode 100644
index 00000000..93a62db6
--- /dev/null
+++ b/digipipe/store/preprocessed/eurostat_lau/scripts/create.py
@@ -0,0 +1,22 @@
+import pandas as pd
+
+
+def process() -> None:
+ cfg = snakemake.config
+ laus = pd.read_excel(
+ snakemake.input[0],
+ **cfg["excel_file"],
+ engine="openpyxl",
+ )
+
+ # Rename columns
+ laus.columns = cfg["column_names"]
+ laus.set_index("lau_code", inplace=True)
+ laus.dropna(inplace=True)
+
+ # Dump
+ laus.to_csv(snakemake.output[0])
+
+
+if __name__ == "__main__":
+ process()
diff --git a/digipipe/store/preprocessed/module.smk b/digipipe/store/preprocessed/module.smk
new file mode 100644
index 00000000..822041c0
--- /dev/null
+++ b/digipipe/store/preprocessed/module.smk
@@ -0,0 +1,103 @@
+"""
+Dataset registry for preprocessed datasets module which is loaded by main
+snakemake file. All datasets in the preprocessed category must be added to
+this file.
+
+Template:
+---------
+module :
+ snakefile: "/create.smk"
+ config: config["store"]["preprocessed"][""]
+use rule * from as preprocessed__*
+
+"""
+
+module bkg_vg250:
+ snakefile: "bkg_vg250/create.smk"
+ config: config["store"]["preprocessed"]["bkg_vg250"]
+use rule * from bkg_vg250 as preprocessed_bkg_vg250_*
+
+module osm_filtered:
+ snakefile: "osm_filtered/create.smk"
+ config: config["store"]["preprocessed"]["osm_filtered"]
+use rule * from osm_filtered as preprocessed_osm_filtered_*
+
+module bnetza_mastr:
+ snakefile: "bnetza_mastr/create.smk"
+ config: config["store"]["preprocessed"]["bnetza_mastr"]
+use rule * from bnetza_mastr as preprocessed_bnetza_mastr_*
+
+module destatis_gv:
+ snakefile: "destatis_gv/create.smk"
+ config: config["store"]["preprocessed"]["destatis_gv"]
+use rule * from destatis_gv as preprocessed_destatis_gv_*
+
+module stala_st_pop_prog:
+ snakefile: "stala_st_pop_prog/create.smk"
+ config: config["store"]["preprocessed"]["stala_st_pop_prog"]
+use rule * from stala_st_pop_prog as preprocessed_stala_st_pop_prog_*
+
+module stala_st_energy:
+ snakefile: "stala_st_energy/create.smk"
+ config: config["store"]["preprocessed"]["stala_st_energy"]
+use rule * from stala_st_energy as preprocessed_stala_st_energy_*
+
+module demandregio:
+ snakefile: "demandregio/create.smk"
+ config: config["store"]["preprocessed"]["demandregio"]
+use rule * from demandregio as preprocessed_demandregio_*
+
+module ba_employment:
+ snakefile: "ba_employment/create.smk"
+ config: config["store"]["preprocessed"]["ba_employment"]
+use rule * from ba_employment as preprocessed_ba_employment_ *
+
+module bmwk_long_term_scenarios:
+ snakefile: "bmwk_long_term_scenarios/create.smk"
+ config: config["store"]["preprocessed"]["bmwk_long_term_scenarios"]
+use rule * from bmwk_long_term_scenarios as preprocessed_bmwk_long_term_scenarios_ *
+
+module seenergies_peta5:
+ snakefile: "seenergies_peta5/create.smk"
+ config: config["store"]["preprocessed"]["seenergies_peta5"]
+use rule * from seenergies_peta5 as preprocessed_seenergies_peta5_ *
+
+module ageb_energy_balance:
+ snakefile: "ageb_energy_balance/create.smk"
+ config: config["store"]["preprocessed"]["ageb_energy_balance"]
+use rule * from ageb_energy_balance as preprocessed_ageb_energy_balance_ *
+
+module dwd_temperature:
+ snakefile: "dwd_temperature/create.smk"
+ config: config["store"]["preprocessed"]["dwd_temperature"]
+use rule * from dwd_temperature as preprocessed_dwd_temperature_ *
+
+module regiostat:
+ snakefile: "regiostat/create.smk"
+ config: config["store"]["preprocessed"]["regiostat"]
+use rule * from regiostat as preprocessed_regiostat_ *
+
+module eurostat_lau:
+ snakefile: "eurostat_lau/create.smk"
+ config: config["store"]["preprocessed"]["eurostat_lau"]
+use rule * from eurostat_lau as preprocessed_eurostat_lau_ *
+
+module rpg_abw_regional_plan:
+ snakefile: "rpg_abw_regional_plan/create.smk"
+ config: config["store"]["preprocessed"]["rpg_abw_regional_plan"]
+use rule * from rpg_abw_regional_plan as preprocessed_rpg_abw_regional_plan_ *
+
+module rli_pv_wfr:
+ snakefile: "rli_pv_wfr/create.smk"
+ config: config["store"]["preprocessed"]["rli_pv_wfr"]
+use rule * from rli_pv_wfr as preprocessed_rli_pv_wfr_ *
+
+module rpg_abw_pv_roof_potential:
+ snakefile: "rpg_abw_pv_roof_potential/create.smk"
+ config: config["store"]["preprocessed"]["rpg_abw_pv_roof_potential"]
+use rule * from rpg_abw_pv_roof_potential as preprocessed_rpg_abw_pv_roof_potential_ *
+
+module dbfz_biomass_capacity_rel:
+ snakefile: "dbfz_biomass_capacity_rel/create.smk"
+ config: config["store"]["preprocessed"]["dbfz_biomass_capacity_rel"]
+use rule * from dbfz_biomass_capacity_rel as preprocessed_dbfz_biomass_capacity_rel_ *
diff --git a/digipipe/store/preprocessed/osm_filtered/__init__.py b/digipipe/store/preprocessed/osm_filtered/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/osm_filtered/config.yml b/digipipe/store/preprocessed/osm_filtered/config.yml
new file mode 100644
index 00000000..b68620c0
--- /dev/null
+++ b/digipipe/store/preprocessed/osm_filtered/config.yml
@@ -0,0 +1,9 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Tags to include (format: [tag, value]))
+tags:
+ [
+ ["building", "*"],
+ ]
diff --git a/digipipe/store/preprocessed/osm_filtered/create.smk b/digipipe/store/preprocessed/osm_filtered/create.smk
new file mode 100644
index 00000000..38f10ef1
--- /dev/null
+++ b/digipipe/store/preprocessed/osm_filtered/create.smk
@@ -0,0 +1,30 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import (
+ get_abs_dataset_path,
+ create_tag_string_osmium,
+ create_tag_string_ogr
+)
+
+DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "osm_filtered", data_dir=True)
+
+rule convert:
+ """
+ Convert OSM pbf file while filtering for all requested tags. All attributes are retained.
+ """
+ input: get_abs_dataset_path("raw", "osm") / "data" / "germany-230101.osm.pbf"
+ output: DATASET_PATH / "germany-230101_filtered.osm.gpkg"
+ params:
+ temp_file=DATASET_PATH / "germany-230101_filtered.osm.pbf",
+ tags=create_tag_string_osmium(config["tags"])
+ shell:
+ """
+ osmium tags-filter --remove-tags -f osm {input} {params.tags} -o {params.temp_file}
+ ogr2ogr -f GPKG -t_srs EPSG:3035 {output} {params.temp_file}
+ rm {params.temp_file}
+ """
diff --git a/digipipe/store/preprocessed/osm_filtered/data/.gitkeep b/digipipe/store/preprocessed/osm_filtered/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/osm_filtered/dataset.md b/digipipe/store/preprocessed/osm_filtered/dataset.md
new file mode 100644
index 00000000..26b16811
--- /dev/null
+++ b/digipipe/store/preprocessed/osm_filtered/dataset.md
@@ -0,0 +1,7 @@
+# OpenStreetMap gefiltert
+
+OSM data nach bestimmten Tags (s. [config.yml](config.yml) --> `tags`) gefiltert,
+zu LAEA Europe (EPSG:3035) umprojiziert und in ein Geopackage konvertiert.
+
+**Achtung:** Konvertierungs- und Extraktionsprozess benötigt ~50 GB
+Speicherplatz und kann viel Zeit in Anspruch nehmen.
diff --git a/digipipe/store/preprocessed/regiostat/__init__.py b/digipipe/store/preprocessed/regiostat/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/regiostat/config.yml b/digipipe/store/preprocessed/regiostat/config.yml
new file mode 100644
index 00000000..76e169a9
--- /dev/null
+++ b/digipipe/store/preprocessed/regiostat/config.yml
@@ -0,0 +1,39 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Demand industry: Excel file
+demand_ind:
+ excel_file:
+ sheet_name: "43531-01-02-4"
+ usecols: "A:B, C:J"
+ skiprows: 4
+ nrows: 540
+ names:
+ [
+ "lau_code",
+ "name",
+ "total",
+ "lignite",
+ "fuel_oil",
+ "natural_gas",
+ "renewables",
+ "electrical_energy",
+ "heat",
+ "other"
+ ]
+
+# Employment industry: Excel file
+employment_ind:
+ excel_file:
+ sheet_name: "42111-01-04-5"
+ usecols: "A:B, C:D"
+ skiprows: 2
+ nrows: 13922
+ names:
+ [
+ "ags",
+ "name",
+ "companies_ind",
+ "employees_ind"
+ ]
diff --git a/digipipe/store/preprocessed/regiostat/create.smk b/digipipe/store/preprocessed/regiostat/create.smk
new file mode 100644
index 00000000..26d5b4ac
--- /dev/null
+++ b/digipipe/store/preprocessed/regiostat/create.smk
@@ -0,0 +1,44 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+from digipipe.store.preprocessed.regiostat.scripts import create
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "regiostat")
+
+rule extract_demand_ind:
+ """
+ Extract energy demand for industry from Excel files and save to CSV
+ """
+ input: get_abs_dataset_path("raw", "regiostat") / "data" / "43531-01-02-4.xlsx"
+ output:
+ demand_states=DATASET_PATH / "data" / "demand_energy_industry_states.csv",
+ demand_districts=DATASET_PATH/ "data" / "demand_energy_industry_districts.csv"
+ run:
+ create.extract_demand_ind(
+ infile=input[0],
+ outfile_states=output.demand_states,
+ outfile_districts=output.demand_districts,
+ cfg=config
+ )
+
+rule extract_employment_ind:
+ """
+ Extract employment for industry from Excel files and save to CSV
+ """
+ input: get_abs_dataset_path("raw", "regiostat") / "data" / "42111-01-04-5.xlsx"
+ output:
+ employment_states=DATASET_PATH / "data" / "employment_industry_states.csv",
+ employment_districts=DATASET_PATH/ "data" / "employment_industry_districts.csv",
+ employment_muns=DATASET_PATH/ "data" / "employment_industry_muns.csv"
+ run:
+ create.extract_employment_ind(
+ infile=input[0],
+ outfile_states=output.employment_states,
+ outfile_districts=output.employment_districts,
+ outfile_muns=output.employment_muns,
+ cfg=config
+ )
diff --git a/digipipe/store/preprocessed/regiostat/data/.gitkeep b/digipipe/store/preprocessed/regiostat/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/regiostat/dataset.md b/digipipe/store/preprocessed/regiostat/dataset.md
new file mode 100644
index 00000000..5d67355f
--- /dev/null
+++ b/digipipe/store/preprocessed/regiostat/dataset.md
@@ -0,0 +1,24 @@
+# Regionalstatistik (GENESIS)
+
+Enthält Datensätze der statistischen Ämter des Bundes und der Länder aus
+[regiostat](../../raw/regiostat/dataset.md).
+
+## Energieverwendung der Betriebe im Verarbeitenden Gewerbe (43531-01-02-4)
+
+Jahreserhebung ü. die Energieverwendung der Betriebe im verarbeitendem Gewerbe.
+
+Änderungen:
+
+- Dateiformat konvertiert
+- Bundesland-, Kreis und Gemeindewerte extrahiert
+- Energie in TWh konvertiert
+
+## Betriebe, tätige Personen, Bruttoentgelte (42111-01-04-5)
+
+Jahreserhebung ü. Betriebe, tätige Personen und Bruttoentgelte der Betriebe im
+verarbeitendem Gewerbe.
+
+Änderungen:
+
+- Dateiformat konvertiert
+- Bundesland-, Kreis und Gemeindewerte extrahiert
diff --git a/digipipe/store/preprocessed/regiostat/scripts/create.py b/digipipe/store/preprocessed/regiostat/scripts/create.py
new file mode 100644
index 00000000..cedaf385
--- /dev/null
+++ b/digipipe/store/preprocessed/regiostat/scripts/create.py
@@ -0,0 +1,103 @@
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+
+def extract_demand_ind(
+ infile: Path, outfile_states: Path, outfile_districts: Path, cfg: dict
+) -> None:
+ data = pd.read_excel(
+ infile,
+ **cfg["demand_ind"]["excel_file"],
+ )
+ # Drop unnecessary headers
+ data = data.iloc[3:]
+ # Use correct zero value and set unavailable values to NaN
+ data = data.replace("-", 0).replace(".", np.nan)
+
+ # Convert from GJ to MWh
+ data = data.set_index(["lau_code", "name"]).div(3.6).reset_index()
+
+ # Filter: federal states
+ data_states = data.loc[data.lau_code.str.len() == 2].set_index("lau_code")
+ data_states["name"] = data_states["name"].str[2:]
+ data_states.to_csv(outfile_states)
+
+ # Filter: districts
+ data_districts = data.loc[data.lau_code.str.len() == 5].set_index(
+ "lau_code"
+ )
+ data_districts["name"] = data_districts["name"].str[6:]
+ data_districts.to_csv(outfile_districts)
+
+
+def extract_employment_ind(
+ infile: Path,
+ outfile_states: Path,
+ outfile_districts: Path,
+ outfile_muns: Path,
+ cfg: dict,
+) -> None:
+ data = pd.read_excel(
+ infile,
+ **cfg["employment_ind"]["excel_file"],
+ )
+
+ # Drop unnecessary headers
+ data = data.iloc[3:]
+ # Use correct zero value and set unavailable values to NaN
+ data = data.replace("-", 0).replace(".", np.nan)
+
+ # Filter: federal states
+ print("Extracting industrial employees and companies: Federal state level")
+ data_states = data.loc[data.ags.str.len() == 2].set_index("ags")
+ data_states["name"] = data_states["name"].str[2:]
+ data_states.to_csv(outfile_states)
+ print(f" Employees: {data_states.employees_ind.sum()}")
+ print(f" Companies: {data_states.companies_ind.sum()}")
+
+ # Filter: districts
+ print("Extracting industrial employees and companies: District level")
+ data_districts = data.loc[data.ags.str.len() == 5].set_index("ags")
+ data_districts["name"] = data_districts["name"].str[6:]
+ data_districts.to_csv(outfile_districts)
+ print(f" Employees: {data_districts.employees_ind.sum()}")
+ print(f" Companies: {data_districts.companies_ind.sum()}")
+ if (
+ data_districts.employees_ind.sum() != data_states.employees_ind.sum()
+ ) or (
+ data_districts.companies_ind.sum() != data_states.companies_ind.sum()
+ ):
+ print(
+ " WARNING: Numbers do not equal state values, "
+ "probably due to missing values!"
+ )
+
+ # Filter: municipalities
+ # Note: Kreisfreie Städte are not included in muns, so they're inserted
+ # manually
+ print("Extracting industrial employees and companies: Municipal level")
+ data_muns = data.loc[data.ags.str.len() == 8].set_index("ags")
+ data_muns["name"] = data_muns["name"].str[8:]
+
+ missing_districts = data_districts.loc[
+ ~data_districts.index.isin(data_muns.index.str[:5])
+ ]
+ missing_districts.index += "000"
+ data_muns = pd.concat([data_muns, missing_districts], axis=0).sort_index()
+ print(
+ f" {len(missing_districts)} "
+ f"municipalities missing, filled up with districts"
+ )
+ print(f" Employees: {data_muns.employees_ind.sum()}")
+ print(f" Companies: {data_muns.companies_ind.sum()}")
+ if (data_muns.employees_ind.sum() != data_states.employees_ind.sum()) or (
+ data_muns.companies_ind.sum() != data_states.companies_ind.sum()
+ ):
+ print(
+ " WARNING: Numbers do not equal state values, "
+ "probably due to missing values!"
+ )
+
+ data_muns.to_csv(outfile_muns)
diff --git a/digipipe/store/preprocessed/rli_pv_wfr/config.yml b/digipipe/store/preprocessed/rli_pv_wfr/config.yml
new file mode 100644
index 00000000..f876820f
--- /dev/null
+++ b/digipipe/store/preprocessed/rli_pv_wfr/config.yml
@@ -0,0 +1,72 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Files to be extracted from raw dataset
+files_extract:
+ geodata:
+ [
+ "air_traffic_control_system.gpkg",
+ "aviation.gpkg",
+ "biosphere_reserve.gpkg",
+ "drinking_water_protection_area.gpkg",
+ "fauna_flora_habitat.gpkg",
+ "floodplain.gpkg",
+ "forest.gpkg",
+ "grid.gpkg",
+ "industry.gpkg",
+ "landscape_protection_area.gpkg",
+ "less_favoured_areas_agricultural.gpkg",
+ "military.gpkg",
+ "national_park.gpkg",
+ "nature_conservation_area.gpkg",
+ "potentialarea_pv_agriculture_lfa-off.gpkg",
+ "potentialarea_pv_road_railway.gpkg",
+ "railway.gpkg",
+ "road.gpkg",
+ "road_railway-500m.gpkg",
+ "settlement-0m.gpkg",
+ "slope.gpkg",
+ "soil_quality_high.gpkg",
+ "soil_quality_low.gpkg",
+ "special_protection_area.gpkg",
+ "water.gpkg",
+ "wetland_ramsar.gpkg"
+ ]
+ datapackage:
+ [
+ "datapackage/datapackage.json"
+ ]
+ metadata:
+ [
+ # Metadata
+ "metadata/rli_wam_ackerbauliches_ertragspotenzial.json",
+ "metadata/rli_wam_bahnverkehr.json",
+ "metadata/rli_wam_benachteiligte_gebiete.json",
+ "metadata/rli_wam_biosphaerenreservate.json",
+ "metadata/rli_wam_drehfunkfeuer.json",
+ "metadata/rli_wam_ee_anlagen.json",
+ "metadata/rli_wam_faunaflorahabitate.json",
+ "metadata/rli_wam_feuchtgebiete.json",
+ "metadata/rli_wam_gelaendeneigung.json",
+ "metadata/rli_wam_gewaesser.json",
+ "metadata/rli_wam_industrie.json",
+ "metadata/rli_wam_landschaftschutzgebiete.json",
+ "metadata/rli_wam_luftverkehr.json",
+ "metadata/rli_wam_militaergebiete.json",
+ "metadata/rli_wam_nationalparke.json",
+ "metadata/rli_wam_naturschutzgebiete.json",
+ "metadata/rli_wam_potentialflaeche_photovoltaik.json",
+ "metadata/rli_wam_potentialflaeche_windenergie.json",
+ "metadata/rli_wam_pv_energieertrag.json",
+ "metadata/rli_wam_siedlungen.json",
+ "metadata/rli_wam_strassen.json",
+ "metadata/rli_wam_stromnetze.json",
+ "metadata/rli_wam_ueberschwemmungsgebiete.json",
+ "metadata/rli_wam_verwaltungsgrenzen.json",
+ "metadata/rli_wam_vogelschutzgebiete.json",
+ "metadata/rli_wam_waelder.json",
+ "metadata/rli_wam_wahlkreise.json",
+ "metadata/rli_wam_wasserschutzgebiete.json",
+ "metadata/rli_wam_windleistungsdichte.json"
+ ]
diff --git a/digipipe/store/preprocessed/rli_pv_wfr/create.smk b/digipipe/store/preprocessed/rli_pv_wfr/create.smk
new file mode 100644
index 00000000..6e1b50e5
--- /dev/null
+++ b/digipipe/store/preprocessed/rli_pv_wfr/create.smk
@@ -0,0 +1,35 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import json
+import re
+import geopandas as gpd
+from pathlib import Path
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "rli_pv_wfr", data_dir=True)
+
+rule extract:
+ """
+ Extract files: geodata, datapackage and metadata
+ """
+ input:
+ get_abs_dataset_path(
+ "raw", "rli_pv_wfr") / "data" /
+ "rli_pv_windflaechenrechner_geodaten_v1.0.zip"
+ output:
+ [DATASET_PATH / f
+ for files in config["files_extract"].values() for f in files]
+ params:
+ outpath=DATASET_PATH,
+ files_extract=" ".join(
+ [f for files in config["files_extract"].values() for f in files]
+ )
+ shell:
+ """
+ unzip {input} {params.files_extract} -d {params.outpath}
+ """
diff --git a/digipipe/store/preprocessed/rli_pv_wfr/data/.gitkeep b/digipipe/store/preprocessed/rli_pv_wfr/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/rli_pv_wfr/dataset.md b/digipipe/store/preprocessed/rli_pv_wfr/dataset.md
new file mode 100644
index 00000000..cc162cb2
--- /dev/null
+++ b/digipipe/store/preprocessed/rli_pv_wfr/dataset.md
@@ -0,0 +1,7 @@
+# Geodaten PV- und Windflächenrechner
+
+Geodaten aus dem [PV- und Windflächenrechner](https://www.agora-energiewende.de/service/pv-und-windflaechenrechner/),
+extrahiert.
+
+Raw dataset:
+[rli_pv_windflaechenrechner](../../raw/rli_pv_wfr/dataset.md)
diff --git a/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/__init__.py b/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/config.yml b/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/config.yml
new file mode 100644
index 00000000..7bc55a0d
--- /dev/null
+++ b/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/config.yml
@@ -0,0 +1,20 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+layer: rpg_abw_pv_roof_potential
+attributes:
+ {"id": "building_id",
+ "installierte_leistung_kw_sued": "installable_power_kw_south",
+ "ertrag_jahr_mwh_jahr_sued": "energy_annual_mwh_south",
+ "installierte_leistung_kw_nord": "installable_power_kw_north",
+ "ertrag_jahr_mwh_jahr_nord": "energy_annual_mwh_north",
+ "installierte_leistung_kw_ost": "installable_power_kw_east",
+ "ertrag_jahr_mwh_jahr_ost": "energy_annual_mwh_east",
+ "installierte_leistung_kw_west": "installable_power_kw_west",
+ "ertrag_jahr_mwh_jahr_west": "energy_annual_mwh_west",
+ "installierte_leistung_kw_flach": "installable_power_kw_flat",
+ "ertrag_jahr_mwh_jahr_flach": "energy_annual_mwh_flat",
+ "denkmalschutz_beachten": "historic_preservation",
+ "grundfl_qm": "building_area_sqm",
+ "geometry": "geometry"}
diff --git a/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/create.smk b/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/create.smk
new file mode 100644
index 00000000..5c92bbbf
--- /dev/null
+++ b/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/create.smk
@@ -0,0 +1,36 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import geopandas as gpd
+
+from digipipe.scripts.geo import (
+ rename_filter_attributes,
+ reproject_simplify,
+ write_geofile
+)
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "rpg_abw_pv_roof_potential")
+
+rule create:
+ input:
+ get_abs_dataset_path(
+ "raw", "rpg_abw_pv_roof_potential"
+ ) / "data" / "rpg_abw_pv_roof_potential.gpkg"
+ output:
+ DATASET_PATH / "data" / "rpg_abw_pv_roof_potential.gpkg"
+ run:
+ data = reproject_simplify(
+ rename_filter_attributes(
+ gdf=gpd.read_file(input[0]),
+ attrs_mapping=config["attributes"]
+ )
+ )
+ write_geofile(
+ gdf=data,
+ file=output[0],
+ layer_name=config["layer"],
+ )
diff --git a/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/data/.gitkeep b/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/dataset.md b/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/dataset.md
new file mode 100644
index 00000000..c79fa3c8
--- /dev/null
+++ b/digipipe/store/preprocessed/rpg_abw_pv_roof_potential/dataset.md
@@ -0,0 +1,7 @@
+# Dachflächenpotenzial PV-Aufdachanlagen in ABW
+
+Abschätzung der installierten Leistung und des Ertrags von PV-Aufdachanlagen in
+Anhalt-Bitterfeld-Wittenberg der Regionalen Planungsgemeinschaft, reprojizert.
+
+Raw dataset:
+[rpg_abw_pv_roof_potential](../../raw/rpg_abw_pv_roof_potential/dataset.md)
diff --git a/digipipe/store/preprocessed/rpg_abw_regional_plan/__init__.py b/digipipe/store/preprocessed/rpg_abw_regional_plan/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/rpg_abw_regional_plan/config.yml b/digipipe/store/preprocessed/rpg_abw_regional_plan/config.yml
new file mode 100644
index 00000000..55f81e65
--- /dev/null
+++ b/digipipe/store/preprocessed/rpg_abw_regional_plan/config.yml
@@ -0,0 +1,53 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Sachlicher Teilplan Wind 2018: VR/EG
+stp_2018:
+ layer: "stp_2018_vreg"
+ attributes:
+ {"bezeich": "name_1",
+ "bezeich_2": "name_2",
+ "bezeich_3": "name_3",
+ "plan_status": "plan_status",
+ "d_beschluss": "decision_date",
+ "d_genehmigung": "authorization_date",
+ "d_inkraft": "in_force_date",
+ "geometry": "geometry"}
+
+# Sachlicher Teilplan Wind 2027: VR/EG
+stp_2027:
+ # Vorranggebiete
+ vr:
+ layer: "stp_2027_vr"
+ attributes:
+ {"bezeich": "name_1",
+ "bezeich_2": "name_2",
+ "bezeich_3": "name_3",
+ "bezeich_4": "name_4",
+ "plan_status": "plan_status",
+ "d_beschluss": "decision_date",
+ "d_genehmigung": "authorization_date",
+ "d_inkraft": "in_force_date",
+ "geometry": "geometry"}
+ # Repoweringgebiete
+ repowering:
+ layer: "stp_2027_repowering"
+ attributes:
+ {"bezeich": "name_1",
+ "bezeich_2": "name_2",
+ "bezeich_3": "name_3",
+ "bezeich_4": "name_4",
+ "plan_status": "plan_status",
+ "d_beschluss": "decision_date",
+ "d_genehmigung": "authorization_date",
+ "d_inkraft": "in_force_date",
+ "geometry": "geometry"}
+ # Suchraum Wald
+ search_area_forest_area:
+ attributes:
+ {"geometry": "geometry"}
+ # Suchraum Offenland
+ search_area_open_area:
+ attributes:
+ {"geometry": "geometry"}
diff --git a/digipipe/store/preprocessed/rpg_abw_regional_plan/create.smk b/digipipe/store/preprocessed/rpg_abw_regional_plan/create.smk
new file mode 100644
index 00000000..5479f29e
--- /dev/null
+++ b/digipipe/store/preprocessed/rpg_abw_regional_plan/create.smk
@@ -0,0 +1,108 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import geopandas as gpd
+from digipipe.scripts.geo import (
+ rename_filter_attributes,
+ reproject_simplify,
+ write_geofile
+)
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "rpg_abw_regional_plan")
+
+rule create_stp_2018:
+ """
+ Sachlicher Teilplan Wind 2018: Preprocess VR/EG
+ """
+ input:
+ get_abs_dataset_path(
+ "raw", "rpg_abw_regional_plan") / "data" / "stp_2018_vreg.gpkg"
+ output:
+ DATASET_PATH / "data" / "stp_2018_vreg.gpkg"
+ run:
+ data = gpd.read_file(input[0])
+ data = reproject_simplify(
+ rename_filter_attributes(
+ gdf=data,
+ attrs_mapping=config["stp_2018"]["attributes"],
+ )
+ )
+ write_geofile(
+ gdf=data,
+ file=output[0],
+ layer_name=config["stp_2018"]["layer"],
+ )
+
+rule create_stp_2027_plan_intent:
+ """
+ Sachlicher Teilplan Wind 2027: Preprocess plan intent
+ """
+ input:
+ get_abs_dataset_path(
+ "raw", "rpg_abw_regional_plan") / "data" /
+ "stp_2027_ideen_{category}.gpkg"
+ output:
+ DATASET_PATH / "data" / "stp_2027_{category}.gpkg",
+ run:
+ data = gpd.read_file(input[0])
+ data = reproject_simplify(
+ rename_filter_attributes(
+ gdf=data,
+ attrs_mapping=config["stp_2027"]["vr"]["attributes"],
+ )
+ )
+ write_geofile(
+ gdf=data,
+ file=output[0],
+ layer_name=config["stp_2027"]["vr"]["layer"],
+ )
+
+rule create_stp_2027_search_area:
+ """
+ Sachlicher Teilplan Wind 2027: Preprocess search areas
+ """
+ input:
+ get_abs_dataset_path(
+ "raw", "rpg_abw_regional_plan") / "data" /
+ "stp_2027_suchraum.gpkg"
+ output:
+ forest=DATASET_PATH / "data" / "stp_2027_search_area_forest_area.gpkg",
+ open_land=DATASET_PATH / "data" / "stp_2027_search_area_open_area.gpkg"
+ run:
+ # Forest
+ data = gpd.read_file(
+ input[0],
+ layer="suchraum_wald_03032023",
+ )
+ data = reproject_simplify(
+ rename_filter_attributes(
+ gdf=data,
+ attrs_mapping=config[
+ "stp_2027"]["search_area_forest_area"]["attributes"],
+ )
+ )
+ write_geofile(
+ gdf=data,
+ file=output.forest,
+ )
+
+ # Open land
+ data = gpd.read_file(
+ input[0],
+ layer="suchraum_offenland_03032023",
+ )
+ data = data.explode(index_parts=False)
+ data = reproject_simplify(
+ rename_filter_attributes(
+ gdf=data,
+ attrs_mapping=config[
+ "stp_2027"]["search_area_open_area"]["attributes"],
+ )
+ )
+ write_geofile(
+ gdf=data,
+ file=output.open_land,
+ )
diff --git a/digipipe/store/preprocessed/rpg_abw_regional_plan/data/.gitkeep b/digipipe/store/preprocessed/rpg_abw_regional_plan/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/rpg_abw_regional_plan/dataset.md b/digipipe/store/preprocessed/rpg_abw_regional_plan/dataset.md
new file mode 100644
index 00000000..f39b9434
--- /dev/null
+++ b/digipipe/store/preprocessed/rpg_abw_regional_plan/dataset.md
@@ -0,0 +1,7 @@
+# Regionalplan Anhalt-Bitterfeld-Wittenberg
+
+Vorverarbeitete Datensätze aus Teilplänen Wind der Regionalen
+Planungsgemeinschaft Anhalt-Bitterfeld-Wittenberg aus
+[rpg_abw_regional_plan](../../raw/rpg_abw_regional_plan/dataset.md).
+
+In der [config.yml](config.yml) können Einstellungen vorgenommen werden.
diff --git a/digipipe/store/preprocessed/rpg_abw_regional_plan/scripts/create.py b/digipipe/store/preprocessed/rpg_abw_regional_plan/scripts/create.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/seenergies_peta5/__init__.py b/digipipe/store/preprocessed/seenergies_peta5/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/seenergies_peta5/config.yml b/digipipe/store/preprocessed/seenergies_peta5/config.yml
new file mode 100644
index 00000000..071de9db
--- /dev/null
+++ b/digipipe/store/preprocessed/seenergies_peta5/config.yml
@@ -0,0 +1,24 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Files to be extracted from raw dataset
+files_extract:
+ residential:
+ [
+ "HD_2015_res_Peta5_0_1_GJ.tfw",
+ "HD_2015_res_Peta5_0_1_GJ.tif.aux.xml",
+ "HD_2015_res_Peta5_0_1_GJ.tif.vat.cpg",
+ "HD_2015_res_Peta5_0_1_GJ.tif",
+ "HD_2015_res_Peta5_0_1_GJ.tif.ovr",
+ "HD_2015_res_Peta5_0_1_GJ.tif.vat.dbf",
+ ]
+ cts:
+ [
+ "HD_2015_ser_Peta5_0_1_GJ.tfw",
+ "HD_2015_ser_Peta5_0_1_GJ.tif.aux.xml",
+ "HD_2015_ser_Peta5_0_1_GJ.tif.vat.cpg",
+ "HD_2015_ser_Peta5_0_1_GJ.tif",
+ "HD_2015_ser_Peta5_0_1_GJ.tif.ovr",
+ "HD_2015_ser_Peta5_0_1_GJ.tif.vat.dbf",
+ ]
diff --git a/digipipe/store/preprocessed/seenergies_peta5/create.smk b/digipipe/store/preprocessed/seenergies_peta5/create.smk
new file mode 100644
index 00000000..d479788a
--- /dev/null
+++ b/digipipe/store/preprocessed/seenergies_peta5/create.smk
@@ -0,0 +1,35 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+import zipfile
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "seenergies_peta5", data_dir=True)
+
+rule extract:
+ """
+ Extract files from archives
+ """
+ input:
+ residential=get_abs_dataset_path(
+ "raw", "seenergies_peta5") / "data" / "Peta5_0_1_HD_res.zip",
+ cts=get_abs_dataset_path(
+ "raw", "seenergies_peta5") / "data" / "Peta5_0_1_HD_ser.zip"
+ output:
+ residential=[DATASET_PATH / f for f in config["files_extract"]["residential"]],
+ cts=[DATASET_PATH/f for f in config["files_extract"]["cts"]]
+ params:
+ outpath=DATASET_PATH,
+ files_extract_residential=config["files_extract"]["residential"],
+ files_extract_cts=config["files_extract"]["cts"]
+ run:
+ # note: buildin zipfile used as bash unzip doesn't work (pkzip required)
+ with zipfile.ZipFile(input.residential, "r") as zf:
+ for f in params.files_extract_residential:
+ zf.extract(f, params.outpath)
+ with zipfile.ZipFile(input.cts, "r") as zf:
+ for f in params.files_extract_cts:
+ zf.extract(f, params.outpath)
diff --git a/digipipe/store/preprocessed/seenergies_peta5/data/.gitkeep b/digipipe/store/preprocessed/seenergies_peta5/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/seenergies_peta5/dataset.md b/digipipe/store/preprocessed/seenergies_peta5/dataset.md
new file mode 100644
index 00000000..83683455
--- /dev/null
+++ b/digipipe/store/preprocessed/seenergies_peta5/dataset.md
@@ -0,0 +1,6 @@
+# sEEnergies Pan-European Thermal Atlas 5.2 (Peta5)
+
+Wärmebedarf (extrahiert) für Europa 2015 in GJ (1ha Auflösung) für
+
+- Haushalte: Raumwärme und Warmwasser
+- GHD: Raumwärme, Warmwasser und Prozesswärme
diff --git a/digipipe/store/preprocessed/stala_st_energy/config.yml b/digipipe/store/preprocessed/stala_st_energy/config.yml
new file mode 100644
index 00000000..320e484f
--- /dev/null
+++ b/digipipe/store/preprocessed/stala_st_energy/config.yml
@@ -0,0 +1,15 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Demand industry: Excel file
+electricity_demand_ind:
+ excel_file:
+ sheet_name: "Stromverbr. nach KR und Jahren"
+ usecols: "A:T"
+ skiprows: 2
+ nrows: 15
+
+ # Years to be included (leave empty for all)
+ years:
+ []
diff --git a/digipipe/store/preprocessed/stala_st_energy/create.smk b/digipipe/store/preprocessed/stala_st_energy/create.smk
new file mode 100644
index 00000000..1b65abc1
--- /dev/null
+++ b/digipipe/store/preprocessed/stala_st_energy/create.smk
@@ -0,0 +1,42 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+import pandas as pd
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path(
+ "preprocessed", "stala_st_energy", data_dir=True
+)
+
+rule extract_power_demand_ind:
+ """
+ Extract industrial electricity demand from Excel file and save to CSV
+ """
+ input:
+ get_abs_dataset_path("raw", "stala_st_energy") / "data" /
+ "Stromverbrauch_nach_Kreisen_ab_Jahr_2003.xlsx"
+ output: DATASET_PATH / "power_demand_industry_st_districts.csv"
+ run:
+ data = pd.read_excel(
+ input[0],
+ **config["electricity_demand_ind"]["excel_file"],
+ engine="openpyxl",
+ ).rename(columns={
+ "Kreisfreie Stadt\nLandkreis\n\nLand": "name"
+ }).set_index("name")
+ data.columns = [int(_[1]) for _ in data.columns.str.split("\n")]
+
+ # Select desired years
+ print(
+ f"Available years for industrial demand: "
+ f"{data.columns.to_list()}"
+ )
+ years = config["electricity_demand_ind"]["years"]
+ if len(years) > 0:
+ data = data[years]
+ print(f" Selected: {years}")
+
+ data.to_csv(output[0])
diff --git a/digipipe/store/preprocessed/stala_st_energy/data/.gitkeep b/digipipe/store/preprocessed/stala_st_energy/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/stala_st_energy/dataset.md b/digipipe/store/preprocessed/stala_st_energy/dataset.md
new file mode 100644
index 00000000..85c3a1b8
--- /dev/null
+++ b/digipipe/store/preprocessed/stala_st_energy/dataset.md
@@ -0,0 +1,13 @@
+# Energiedaten Sachsen-Anhalt
+
+Datensätze zur Energie- und Wasserversorgung des Statistischen Landesamtes
+Sachsen-Anhalt, extrahiert und konvertiert.
+
+## Daten
+
+Stromverbrauch der Industriebetriebe nach Kreisen 2003-2021 in MWh
+
+- Datei: `power_demand_industry_st_districts.csv`
+
+Raw dataset:
+[stala_st_energy](../../raw/stala_st_energy/dataset.md)
diff --git a/digipipe/store/preprocessed/stala_st_energy/metadata.json b/digipipe/store/preprocessed/stala_st_energy/metadata.json
new file mode 100644
index 00000000..86df8626
--- /dev/null
+++ b/digipipe/store/preprocessed/stala_st_energy/metadata.json
@@ -0,0 +1,5 @@
+{
+ "Datenquellen": {
+ "Stromverbrauch der Industriebetriebe nach Kreisen": "https://statistik.sachsen-anhalt.de/themen/wirtschaftsbereiche/energie-und-wasserversorgung/tabellen-energieverwendung#c206986"
+ }
+}
diff --git a/digipipe/store/preprocessed/stala_st_energy/scripts/create.py b/digipipe/store/preprocessed/stala_st_energy/scripts/create.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/stala_st_pop_prog/__init__.py b/digipipe/store/preprocessed/stala_st_pop_prog/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/stala_st_pop_prog/config.yml b/digipipe/store/preprocessed/stala_st_pop_prog/config.yml
new file mode 100644
index 00000000..70541393
--- /dev/null
+++ b/digipipe/store/preprocessed/stala_st_pop_prog/config.yml
@@ -0,0 +1,18 @@
+##############################################################################
+# This file holds configuration parameters for the creation of this dataset. #
+##############################################################################
+
+# Excel file
+excel_file:
+ sheet_name: "7. RBP"
+ usecols: "A, G:S"
+ skiprows: 3
+ nrows: 230
+ dtype:
+ {
+ "AGS": str
+ }
+
+# Years to be included (leave empty for all)
+years:
+ []
diff --git a/digipipe/store/preprocessed/stala_st_pop_prog/create.smk b/digipipe/store/preprocessed/stala_st_pop_prog/create.smk
new file mode 100644
index 00000000..114be7ff
--- /dev/null
+++ b/digipipe/store/preprocessed/stala_st_pop_prog/create.smk
@@ -0,0 +1,17 @@
+"""
+Snakefile for this dataset
+
+Note: To include the file in the main workflow, it must be added to the respective module.smk .
+"""
+
+from digipipe.store.utils import get_abs_dataset_path
+
+DATASET_PATH = get_abs_dataset_path("preprocessed", "stala_st_pop_prog")
+
+rule create:
+ """
+ Extract municipality population data from Excel files and save to CSVs
+ """
+ input: get_abs_dataset_path("raw", "stala_st_pop_prog") / "data" / "1_Internettabelle_7RBP_nach_Prognosejahr_Geschlecht_alle_Ebenen.xlsx"
+ output: DATASET_PATH / "data" / "population_prognosis_st_muns.csv"
+ script: DATASET_PATH / "scripts" / "create.py"
diff --git a/digipipe/store/preprocessed/stala_st_pop_prog/data/.gitkeep b/digipipe/store/preprocessed/stala_st_pop_prog/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/preprocessed/stala_st_pop_prog/dataset.md b/digipipe/store/preprocessed/stala_st_pop_prog/dataset.md
new file mode 100644
index 00000000..b2e56254
--- /dev/null
+++ b/digipipe/store/preprocessed/stala_st_pop_prog/dataset.md
@@ -0,0 +1,7 @@
+# Bevölkerungsprognose Sachsen-Anhalt
+
+Bevölkerungsprognose je Gemeinde bis 2035 des Statistischen Landesamtes
+Sachsen-Anhalt, extrahiert und konvertiert.
+
+Raw dataset:
+[stala_st_pop_prog](../../raw/stala_st_pop_prog/dataset.md)
diff --git a/digipipe/store/preprocessed/stala_st_pop_prog/scripts/create.py b/digipipe/store/preprocessed/stala_st_pop_prog/scripts/create.py
new file mode 100644
index 00000000..1579c00f
--- /dev/null
+++ b/digipipe/store/preprocessed/stala_st_pop_prog/scripts/create.py
@@ -0,0 +1,35 @@
+import pandas as pd
+
+
+def process() -> None:
+ data = pd.read_excel(
+ snakemake.input[0],
+ **snakemake.config["excel_file"],
+ engine="openpyxl",
+ ).rename(columns={"AGS": "ags"})
+
+ # Rename columns
+ data.set_index("ags", inplace=True)
+ data.columns = [int(col[1]) for col in data.columns.str.split("\n")]
+
+ # Select desired years
+ print(f"Available years for population prognosis: {data.columns.to_list()}")
+ years = snakemake.config["years"]
+ if len(years) > 0:
+ data = data[years]
+ print(f" Selected: {years}")
+
+ # Rename columns
+ data.reset_index(inplace=True)
+
+ # Drop non-municipal data
+ data = data.loc[data.ags.str.len() > 2]
+ data = data.assign(
+ ags=data.ags.str.pad(width=8, side="right", fillchar="0")
+ ).set_index("ags")
+
+ data.to_csv(snakemake.output[0])
+
+
+if __name__ == "__main__":
+ process()
diff --git a/digipipe/store/raw/TEMPLATE/data/.gitkeep b/digipipe/store/raw/TEMPLATE/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/TEMPLATE/dataset.md b/digipipe/store/raw/TEMPLATE/dataset.md
new file mode 100644
index 00000000..98ede272
--- /dev/null
+++ b/digipipe/store/raw/TEMPLATE/dataset.md
@@ -0,0 +1,55 @@
+# Name des Datensatzes
+
+Eine kurze Beschreibung des Datensatzes.
+Diese hilft der Dokumentation und bei der Erstellung des Berichts.
+
+# Notizen (nur zur Information, nicht Teil des dataset.md files)
+
+Benennungskonvention: `` (Kleinschreibung), z.B. ein
+Datensatz zu Naturschutzgebieten des Bundesamtes für Naturschutz (BfN) könnte
+`bfn_natural_reserves` heißen.
+
+Was ist ein Datensatz? Es gibt verschiedene Definitionen, aber in dieser
+Pipeline ist ein Datensatz eine Sammlung von Daten, die als eine Einheit
+behandelt, welche aus mehreren Dateien bestehen und durch eine einzige
+Metadaten-Datei identifiziert werden kann.
+
+Beispiele:
+- [OSM Germany](https://download.geofabrik.de/europe/germany-latest.osm.pbf)
+- [ERA5 weather dataset](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview)
+- [BKG administrative areas](https://gdz.bkg.bund.de/index.php/default/verwaltungsgebiete-1-250-000-stand-01-01-vg250-01-01.html)
+
+Rohdateien kommen in das Verzeichnis `data` und werden nach Möglichkeit nicht
+umbenannt.
+
+## Beschreibung
+
+Bitte gib zumindest eine kurze Beschreibung:
+
+- Worum geht es in dem Datensatz
+- Gibt es Besonderheiten, die es zu wissen gilt? (neben Metadaten, welche
+ UNBEDINGT erstellt werden müssen, dazu unten mehr)
+
+Eine schnelle und suboptimale Beschreibung ist besser als keine.
+
+## Metadaten
+
+Füge für jeden Roh-/Originaldatensatz, der erstellt wird, Metadaten zur
+Beschreibung der Daten mit maschinenlesbaren Informationen hinzu.
+Folge der [OEP](https://openenergy-platform.org/about/) Metadaten v1.5.1.
+Es kann der [Metadata creator](https://meta.rl-institut.de/meta_creator/151)
+verwendet werden.
+
+Zum Vergleich die [metadata.json](metadata.json) in diesem Verzeichnis.
+
+Alternativ kann sie auch manuell erstellt werden:
+Folgen Sie
+[dieses Beispiel](https://github.com/OpenEnergyPlatform/oemetadata/blob/develop/metadata/latest/example.json)
+um zu verstehen, wie die Felder verwendet werden. Die Felder werden in der
+[Open Energy Metadata Description](https://github.com/OpenEnergyPlatform/oemetadata/blob/develop/metadata/v141/metadata_key_description.md)
+im Detail beschrieben. Bitte überprüfe, ob der Metadatenstring den OEP-Metadaten
+Standards entspricht, indem das
+[OMI-Tool](https://github.com/OpenEnergyPlatform/omi) verwendet wird.
+Wenn der Metadatenstring konform ist, bringt OMI die Schlüssel in die richtige
+Reihenfolge und gibt den vollständigen String aus (für den Export die Option
+`-o` verwenden).
diff --git a/digipipe/store/raw/TEMPLATE/metadata.json b/digipipe/store/raw/TEMPLATE/metadata.json
new file mode 100644
index 00000000..0802ee07
--- /dev/null
+++ b/digipipe/store/raw/TEMPLATE/metadata.json
@@ -0,0 +1,196 @@
+{
+ "name": null,
+ "title": null,
+ "id": null,
+ "description": null,
+ "language": [
+ null
+ ],
+ "subject": [
+ {
+ "name": null,
+ "path": null
+ }
+ ],
+ "keywords": [
+ null
+ ],
+ "publicationDate": null,
+ "context": {
+ "homepage": null,
+ "documentation": null,
+ "sourceCode": null,
+ "contact": null,
+ "grantNo": null,
+ "fundingAgency": null,
+ "fundingAgencyLogo": null,
+ "publisherLogo": null
+ },
+ "spatial": {
+ "location": null,
+ "extent": null,
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": null,
+ "timeseries": [
+ {
+ "start": null,
+ "end": null,
+ "resolution": null,
+ "alignment": null,
+ "aggregationType": null
+ },
+ {
+ "start": null,
+ "end": null,
+ "resolution": null,
+ "alignment": null,
+ "aggregationType": null
+ }
+ ]
+ },
+ "sources": [
+ {
+ "title": null,
+ "description": null,
+ "path": null,
+ "licenses": [
+ {
+ "name": null,
+ "title": null,
+ "path": null,
+ "instruction": null,
+ "attribution": null
+ }
+ ]
+ },
+ {
+ "title": null,
+ "description": null,
+ "path": null,
+ "licenses": [
+ {
+ "name": null,
+ "title": null,
+ "path": null,
+ "instruction": null,
+ "attribution": null
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": null,
+ "title": null,
+ "path": null,
+ "instruction": null,
+ "attribution": null
+ }
+ ],
+ "contributors": [
+ {
+ "title": null,
+ "email": null,
+ "date": null,
+ "object": null,
+ "comment": null
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [
+ {
+ "name": null,
+ "description": null,
+ "type": null,
+ "unit": null,
+ "isAbout": [
+ {
+ "name": null,
+ "path": null
+ }
+ ],
+ "valueReference": [
+ {
+ "value": null,
+ "name": null,
+ "path": null
+ }
+ ]
+ },
+ {
+ "name": null,
+ "description": null,
+ "type": null,
+ "unit": null,
+ "isAbout": [
+ {
+ "name": null,
+ "path": null
+ }
+ ],
+ "valueReference": [
+ {
+ "value": null,
+ "name": null,
+ "path": null
+ }
+ ]
+ }
+ ],
+ "primaryKey": [
+ null
+ ],
+ "foreignKeys": [
+ {
+ "fields": [
+ null
+ ],
+ "reference": {
+ "resource": null,
+ "fields": [
+ null
+ ]
+ }
+ }
+ ]
+ },
+ "dialect": {
+ "delimiter": null,
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": null,
+ "review": {
+ "path": null,
+ "badge": null
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/ageb_energy_balance/data/.gitkeep b/digipipe/store/raw/ageb_energy_balance/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/ageb_energy_balance/dataset.md b/digipipe/store/raw/ageb_energy_balance/dataset.md
new file mode 100644
index 00000000..8a90c7fa
--- /dev/null
+++ b/digipipe/store/raw/ageb_energy_balance/dataset.md
@@ -0,0 +1,5 @@
+# AGEB – Anwendungsbilanzen für die Endenergiesektoren 2011 bis 2021
+
+Detaillierte Anwendungsbilanzen der Endenergiesektoren für 2020 und 2021 sowie
+zusammenfassende Zeitreihen zum Endenergieverbrauch nach Energieträgern und
+Anwendungszwecken für Jahre von 2011 bis 2021 der AG Energiebilanzen.
diff --git a/digipipe/store/raw/ageb_energy_balance/metadata.json b/digipipe/store/raw/ageb_energy_balance/metadata.json
new file mode 100644
index 00000000..fae573f4
--- /dev/null
+++ b/digipipe/store/raw/ageb_energy_balance/metadata.json
@@ -0,0 +1,78 @@
+{
+ "name": "ageb_energy_balance",
+ "title": "AGEB – Anwendungsbilanzen für die Endenergiesektoren 2011 bis 2021",
+ "id": "ageb_energy_balance",
+ "description": "Detaillierte Anwendungsbilanzen der Endenergiesektoren für 2020 und 2021 sowie zusammenfassende Zeitreihen zum Endenergieverbrauch nach Energieträgern und Anwendungszwecken für Jahre von 2011 bis 2021",
+ "language": [
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "Endenergiesektoren",
+ "Anwendungsbilanzen",
+ "energy-balance"
+ ],
+ "publicationDate": "2022-12-01",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": "2022-12-01",
+ "timeseries": null
+ },
+ "sources": [
+ {
+ "title": "AGEB – Anwendungsbilanzen für die Endenergiesektoren 2011 bis 2021",
+ "description": "Detaillierte Anwendungsbilanzen der Endenergiesektoren für 2020 und 2021 sowie zusammenfassende Zeitreihen zum Endenergieverbrauch nach Energieträgern und Anwendungszwecken für Jahre von 2011 bis 2021",
+ "path": "https://ag-energiebilanzen.de/daten-und-fakten/anwendungsbilanzen/",
+ "licenses": null
+ }
+ ],
+ "licenses": null,
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-08-15",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": null,
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/ba_employment/data/.gitkeep b/digipipe/store/raw/ba_employment/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/ba_employment/dataset.md b/digipipe/store/raw/ba_employment/dataset.md
new file mode 100644
index 00000000..88d837d5
--- /dev/null
+++ b/digipipe/store/raw/ba_employment/dataset.md
@@ -0,0 +1,5 @@
+# Sozialversicherungspflichtig Beschäftigte und Betriebe
+
+Gemeindedaten der sozialversicherungspflichtig Beschäftigten am 30.06.2022 nach
+Wohn- und Arbeitsort - Deutschland, Länder, Kreise und Gemeinden (Jahreszahlen)
+der Bundesagentur für Arbeit.
diff --git a/digipipe/store/raw/ba_employment/metadata.json b/digipipe/store/raw/ba_employment/metadata.json
new file mode 100644
index 00000000..802556f6
--- /dev/null
+++ b/digipipe/store/raw/ba_employment/metadata.json
@@ -0,0 +1,113 @@
+{
+ "name": "ba_employment",
+ "title": "Gemeindedaten der sozialversicherungspflichtig Beschäftigten nach Wohn- und Arbeitsort",
+ "id": "ba_employment",
+ "description": "Zahl der soziaversicherungspflichtig Beschäftigten nach: Wohnort, Personengruppen, Arbeitsort, Wohnort gleich Arbeitsort, Einpendeler, Auspendler, Zahl der Betriebe",
+ "language": [
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "Gemeindedaten",
+ "sozialversicherungspflichtig",
+ "Beschäftigte",
+ "Wohnort",
+ "Arbeitsort"
+ ],
+ "publicationDate": "2023-01-16",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": "2022-06-23",
+ "timeseries": null
+ },
+ "sources": [
+ {
+ "title": "Gemeindedaten der sozialversicherungspflichtig Beschäftigten nach Wohn- und Arbeitsort",
+ "description": "Zahl der soziaversicherungspflichtig Beschäftigten nach: Wohnort, Personengruppen, Arbeitsort, Wohnort gleich Arbeitsort, Einpendeler, Auspendler, Zahl der Betriebe",
+ "path": "https://statistik.arbeitsagentur.de/SiteGlobals/Forms/Suche/Einzelheftsuche_Formular.html?nn=15024&topic_f=beschaeftigung-sozbe-gemband",
+ "licenses": [
+ {
+ "name": null,
+ "title": null,
+ "path": "https://statistik.arbeitsagentur.de/DE/Statischer-Content/Servicebereich-Navigation/Bezugsbedingungen.html?nn=6654",
+ "instruction": "Sie können Informationen speichern, (auch auszugsweise) mit Quellenangabe weitergeben, vervielfältigen und verbreiten. Die Inhalte dürfen nicht verändert oder verfälscht werden. Eigene Berechnungen sind erlaubt, jedoch als solche kenntlich zu machen. Im Falle einer Zugänglichmachung im Internet soll dies in Form einer Verlinkung auf die Homepage der Statistik der Bundesagentur für Arbeit erfolgen. Die Nutzung der Inhalte für gewerbliche Zwecke, ausgenommen Presse, Rundfunk und Fernsehen und wissenschaftliche Publikationen, bedarf der Genehmigung durch die Statistik der Bundesagentur für Arbeit.",
+ "attribution": "© Statistik der Bundesagentur für Arbeit"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": null,
+ "title": null,
+ "path": "https://statistik.arbeitsagentur.de/DE/Statischer-Content/Servicebereich-Navigation/Bezugsbedingungen.html?nn=6654",
+ "instruction": "Sie können Informationen speichern, (auch auszugsweise) mit Quellenangabe weitergeben, vervielfältigen und verbreiten. Die Inhalte dürfen nicht verändert oder verfälscht werden. Eigene Berechnungen sind erlaubt, jedoch als solche kenntlich zu machen. Im Falle einer Zugänglichmachung im Internet soll dies in Form einer Verlinkung auf die Homepage der Statistik der Bundesagentur für Arbeit erfolgen. Die Nutzung der Inhalte für gewerbliche Zwecke, ausgenommen Presse, Rundfunk und Fernsehen und wissenschaftliche Publikationen, bedarf der Genehmigung durch die Statistik der Bundesagentur für Arbeit.",
+ "attribution": "© Statistik der Bundesagentur für Arbeit"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-08-15",
+ "object": "metadata",
+ "comment": "Create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": [],
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/bkg_vg250/__init__.py b/digipipe/store/raw/bkg_vg250/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/bkg_vg250/data/.gitkeep b/digipipe/store/raw/bkg_vg250/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/bkg_vg250/dataset.md b/digipipe/store/raw/bkg_vg250/dataset.md
new file mode 100644
index 00000000..7edec535
--- /dev/null
+++ b/digipipe/store/raw/bkg_vg250/dataset.md
@@ -0,0 +1,3 @@
+# Verwaltungsgebiete Deutschlands
+
+Verwaltungsgebiete Deutschlands (Verwaltungsgebiete 1:250 000).
diff --git a/digipipe/store/raw/bkg_vg250/metadata.json b/digipipe/store/raw/bkg_vg250/metadata.json
new file mode 100644
index 00000000..6cee4857
--- /dev/null
+++ b/digipipe/store/raw/bkg_vg250/metadata.json
@@ -0,0 +1,102 @@
+{
+ "name": "bkg_vg250",
+ "title": "Adminstrative areas of Germany",
+ "id": "bkg_vb250",
+ "description": "Geopackage with administative areas of Germany - Verwaltungsgebiete 1:250 000",
+ "language": [
+ "en-GB",
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "adminstrative areas",
+ "Verwaltungsgebiete"
+ ],
+ "publicationDate": "2022-01-01",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": "1:250 000"
+ },
+ "temporal": {
+ "referenceDate": "2022-01-01",
+ "timeseries": null
+ },
+"sources": [
+ {
+ "title": "Bundesamt für Kartographie und Geodäsie - Verwaltungsgebiete 1:250 000 VG250 (Ebenen)",
+ "description": "Dieser Datensatz stellt die Verwaltungsgebiete 1:250 000 (VG250) mit Stand 01.01. für das Gebiet der Bundesrepublik Deutschland bereit.",
+ "path": "https://gdz.bkg.bund.de/index.php/default/digitale-geodaten/verwaltungsgebiete/verwaltungsgebiete-1-250-000-stand-01-01-vg250-01-01.html",
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Open Data Datenlizenz Deutschland – Namensnennung – Version 2.0",
+ "path": "http://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets;be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": " © GeoBasis-DE / BKG - 2022"
+ }
+ ]
+ }
+ ],
+ "contributors": [
+ {
+ "title": "hedwiglieselotte",
+ "email": "hedwig.bartels@rl-institut.de",
+ "date": "2023-03-23",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": [],
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/bmwk_long_term_scenarios/data/.gitkeep b/digipipe/store/raw/bmwk_long_term_scenarios/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/bmwk_long_term_scenarios/dataset.md b/digipipe/store/raw/bmwk_long_term_scenarios/dataset.md
new file mode 100644
index 00000000..23a4f2ed
--- /dev/null
+++ b/digipipe/store/raw/bmwk_long_term_scenarios/dataset.md
@@ -0,0 +1,44 @@
+# BMWK Langfristszenarien
+
+Langfristszenarien des Bundesministerium für Wirtschaft und Klimaschutz, Daten
+auf Deutschlandebene.
+
+Die Daten wurden über den
+[Szenario Explorer](https://langfristszenarien.de/enertile-explorer-de/szenario-explorer/)
+abgerufen.
+
+## Verwendete Szenarien
+
+- **T45-Strom:** Stromfokussiertes Szenario aus den T45-Szenarien aus 2023, die
+ Wege zur Treibhausgasneutralität bis 2045 unter Einhaltung aktueller
+ politischer Vorgaben erreichen. Die Daten dieses Szenarios werden als
+ Grundlage für das Zielszenario in der Region verwendet.
+- **TN-Strom:** Stromfokussiertes Szenario aus den TN-Szenarien aus 2021, die
+ unterschiedliche Pfade für Deutschland mit dem Ziel treibhausgasneutral bis
+ 2050 zu werden. Die Daten dieses Szenarios werden als Grundlage für den
+ Status quo verwendet (Ausnahme: Erzeugung Wärmenetze, hier wurden manuell
+ Daten für 2021 ergänzt).
+
+## Daten
+
+### T45-Strom
+
+| Datensatz | Quelle | Datei |
+|------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------|
+| Gebäude: Haushalte und GHD Energiebedarf | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/51944/21559a9532131c061668bf0751e519e3) | `T45-Strom_buildings_heating_demand_by_carrier.csv` |
+| Gebäude: Anzahl der Heizungen nach Technologie | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/51944/21559a9532131c061668bf0751e519e3) | `T45-Strom_buildings_heating_structure_by_technology.csv` |
+| GHD Energieträger | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/52700/c6980ea467bb26a922d34617b4fd4798) | `T45-Strom_cts_demand.csv` |
+| Haushalte Energieträger | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/52700/c6980ea467bb26a922d34617b4fd4798) | `T45-Strom_hh_demand.csv` |
+| Industrie Energiebedarf | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/52612/9de48084ac2d54c418daaf02a6ee26e0) | `T45-Strom_ind_demand.csv` |
+| Stromsystem Deutschland Leistung | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/48766/5c11999a03c547e04e73d61e4b5fc633) | `T45-Strom_electricity_installed_power.csv` |
+| Erzeugung Wärmenetze Deutschland | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/49949/cf898070daec6a4e613dc889927a5feb), [Link2](https://static.agora-energiewende.de/fileadmin/Projekte/2022/2022-11_DE_Large_Scale_Heatpumps/A-EW_293_Rollout_Grosswaermepumpen_WEB.pdf) (S. 37) | `T45-Strom_Generation_Heatgrids_Germany.csv` |
+
+### TN-Strom
+
+| Datensatz | Quelle | Datei |
+|------------------------------------------------|-----------------------------------------------------------------------------------------------------------|----------------------------------------------------------|
+| Gebäude: Haushalte und GHD Energiebedarf | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/8198/698cee83d667a2f44fdea7e78ee799a2) | `TN-Strom_buildings_heating_demand_by_carrier.csv` |
+| Gebäude: Anzahl der Heizungen nach Technologie | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/8198/698cee83d667a2f44fdea7e78ee799a2) | `TN-Strom_buildings_heating_structure_by_technology.csv` |
+| GHD Energieträger | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/8660/ae5a14ff0c320cbd31c5eeff2ede54ba) | `TN-Strom_cts_demand.csv` |
+| Haushalte Energieträger | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/8660/ae5a14ff0c320cbd31c5eeff2ede54ba) | `TN-Strom_hh_demand.csv` |
+| Industrie Energiebedarf | [Link](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/29085/084bd7f45f40d31fd53341e6a94f532c) | `TN-Strom_ind_demand.csv` |
diff --git a/digipipe/store/raw/bmwk_long_term_scenarios/metadata.json b/digipipe/store/raw/bmwk_long_term_scenarios/metadata.json
new file mode 100644
index 00000000..b97a11e9
--- /dev/null
+++ b/digipipe/store/raw/bmwk_long_term_scenarios/metadata.json
@@ -0,0 +1,133 @@
+{
+ "name": "bmwk_long_term_scenarios",
+ "title": "BMWK Langfristszenarien",
+ "id": "bmwk_long_term_scenarios",
+ "description": "Langfristszenarien des Bundesministerium für Wirtschaft und Klimaschutz, Daten auf Deutschlandebene.",
+ "language": [
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "BMWK",
+ "Langfristszenario",
+ "T45-Strom",
+ "TN-Strom"
+ ],
+ "publicationDate": null,
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": null,
+ "timeseries": [
+ {
+ "start": null,
+ "end": null,
+ "resolution": null,
+ "alignment": null,
+ "aggregationType": null
+ },
+ {
+ "start": null,
+ "end": null,
+ "resolution": null,
+ "alignment": null,
+ "aggregationType": null
+ }
+ ]
+ },
+ "sources": [
+ {
+ "title": "BMWK Langfristszenarien",
+ "description": "Langfristszenarien des Bundesministerium für Wirtschaft und Klimaschutz, Daten auf Deutschlandebene.",
+ "path": "https://langfristszenarien.de/enertile-explorer-de/szenario-explorer/",
+ "licenses": null
+ },
+ {
+ "title": null,
+ "description": null,
+ "path": null,
+ "licenses": [
+ {
+ "name": null,
+ "title": null,
+ "path": null,
+ "instruction": null,
+ "attribution": null
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": null,
+ "title": null,
+ "path": null,
+ "instruction": null,
+ "attribution": null
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-08-15",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": null,
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.6.0",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/bnetza_mastr/data/.gitkeep b/digipipe/store/raw/bnetza_mastr/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/bnetza_mastr/dataset.md b/digipipe/store/raw/bnetza_mastr/dataset.md
new file mode 100644
index 00000000..3b9b7255
--- /dev/null
+++ b/digipipe/store/raw/bnetza_mastr/dataset.md
@@ -0,0 +1,20 @@
+# Erzeugungsanlagen aus Marktstammdatenregister
+
+Ereugungsanlagen aus dem Markstammdatenregister, das mit dem Tool
+[open-mastr](https://github.com/OpenEnergyPlatform/open-MaStR) erstellt und
+abgelegt wurde. Die Daten wurden folgendermaßen erstellt:
+```
+from open_mastr import Mastr
+db = Mastr()
+db.download("bulk")
+db.to_csv(None) # (None for all data)
+```
+
+Die abgelegten CSV-Dateien (alle Tabellen) wurden um einen benutzerdefinierten
+Export von Speichereinheiten mit
+`sqlite3 -header -csv -separator "," open-mastr.db "select * from storage_units;" > bnetza_mastr_storage_unit_raw.csv`
+erweitert. Anschließend wurden alle Dateien komprimiert.
+
+Das Marktstammdatenregister (MaStR) ist ein deutsches Register, welches von der
+Bundesnetzagentur (BNetza) bereitgestellt wird und alle in Deutschland
+befindlichen Strom- und Gasanlagen erfasst.
diff --git a/digipipe/store/raw/bnetza_mastr/metadata.json b/digipipe/store/raw/bnetza_mastr/metadata.json
new file mode 100644
index 00000000..ad353d26
--- /dev/null
+++ b/digipipe/store/raw/bnetza_mastr/metadata.json
@@ -0,0 +1,112 @@
+{
+ "name": "bnetza_mastr",
+ "title": "Marktstammdatenregisterdaten",
+ "id": "bnetza_mastr",
+ "description": "Daten aus dem Marktstammdatenregister der Bundesnetzagentur",
+ "language": [
+ "en-GB",
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "Markstammdatenregister",
+ "openmastr",
+ "mastr"
+ ],
+ "publicationDate": "2022-12-19",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": "2022-12-19",
+ "timeseries": null
+ },
+ "sources": [
+ {
+ "title": "Marktstammdatenregister",
+ "description": "Marktstammdatenregister der Bundesnetzagentur Deutschland",
+ "path": "https://www.marktstammdatenregister.de/MaStR/Datendownload",
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Open Data Datenlizenz Deutschland – Namensnennung – Version 2.0",
+ "path": "http://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets;be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Marktstammdatenregister 2023"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Open Data Datenlizenz Deutschland – Namensnennung – Version 2.0?" ,
+ "path": "http://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets;be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Marktstammdatenregister 2023"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "hedwiglieselotte",
+ "email": "hedwig.bartels@rl-institut.de",
+ "date": "2023-03-28",
+ "object": "metadata",
+ "comment": "Create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": [],
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/bnetza_mastr_correction_region/data/.gitkeep b/digipipe/store/raw/bnetza_mastr_correction_region/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/bnetza_mastr_correction_region/dataset.md b/digipipe/store/raw/bnetza_mastr_correction_region/dataset.md
new file mode 100644
index 00000000..734e6ef5
--- /dev/null
+++ b/digipipe/store/raw/bnetza_mastr_correction_region/dataset.md
@@ -0,0 +1,30 @@
+# Marktstammdatenregister Datenkorrektur PV
+
+Überprüfung und manuelle Datenkorrektur der Photovoltaikanlagen aus dem
+prozessierten Marktstammdatenregister (Datensatz:
+[bnetza_mastr](../bnetza_mastr/dataset.md)).
+
+## Plausibiltätsprüfung
+
+Um grobe Fehler herauszufiltern wird überprüft, ob
+
+- Anlage in Betrieb ist (status = "In Betrieb"),
+- Anlage Strom produziert,
+- Brutto- und Nettokapazität plausibel sind und
+- die Kategorisierung, d.h. Zuordnung eine PV-Anlage zu Freifläche oder Dach,
+ plausibel ist (manuelle, visuelle Prüfung von geolokalisierten
+ PV-Aufdachanlagen anhand von
+ [Orthofotos](https://www.geodatenportal.sachsen-anhalt.de/wss/service/ST_LVermGeo_DOP_WMS_OpenData/guest))
+
+## Dateien
+
+- Korrektur Freiflächenanlagen `bnetza_mastr_pv_ground_region_correction.ods`
+- Korrektur Aufdachanlagen `bnetza_mastr_pv_roof_region_correction.ods`
+
+mit Spalten:
+
+- _mastr_id_: ID aus dem MaStR
+- _reason_: Fehler (wrong_type, wrong_position)
+- _wrong_attr_: Fehlerhaftes Attribut
+- _correction_: Korrigierter Attributwert (None, wenn Korrektur nicht möglich).
+ Korrigierte Geometrien liegen in EPSG:3035 vor.
diff --git a/digipipe/store/raw/bnetza_mastr_correction_region/metadata.json b/digipipe/store/raw/bnetza_mastr_correction_region/metadata.json
new file mode 100644
index 00000000..b36e1d6d
--- /dev/null
+++ b/digipipe/store/raw/bnetza_mastr_correction_region/metadata.json
@@ -0,0 +1,112 @@
+{
+ "name": "bnetza_mastr_correction",
+ "title": "Marktstammdatenregisterdaten - Manuelle Korrektur",
+ "id": "bnetza_mastr",
+ "description": "Daten aus dem Marktstammdatenregister der Bundesnetzagentur",
+ "language": [
+ "en-GB",
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "Markstammdatenregister",
+ "openmastr",
+ "mastr"
+ ],
+ "publicationDate": "2022-12-19",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": "2022-12-19",
+ "timeseries": null
+ },
+ "sources": [
+ {
+ "title": "Marktstammdatenregister",
+ "description": "Marktstammdatenregister der Bundesnetzagentur Deutschland",
+ "path": "https://www.marktstammdatenregister.de/MaStR/Datendownload",
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Open Data Datenlizenz Deutschland – Namensnennung – Version 2.0",
+ "path": "http://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets;be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Marktstammdatenregister 2023"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Open Data Datenlizenz Deutschland – Namensnennung – Version 2.0?" ,
+ "path": "http://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets;be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Marktstammdatenregister 2023"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "hedwiglieselotte",
+ "email": "hedwig.bartels@rl-institut.de",
+ "date": "2023-03-28",
+ "object": "metadata",
+ "comment": "Create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": [],
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/dbfz_biomass_heat_capacities/data/.gitkeep b/digipipe/store/raw/dbfz_biomass_heat_capacities/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/dbfz_biomass_heat_capacities/dataset.md b/digipipe/store/raw/dbfz_biomass_heat_capacities/dataset.md
new file mode 100644
index 00000000..aaf19ff9
--- /dev/null
+++ b/digipipe/store/raw/dbfz_biomass_heat_capacities/dataset.md
@@ -0,0 +1,38 @@
+# Installierte Leistungen von Biomasse-Konversionstechnologien
+
+Die installierten Leistungen in MW wird im Szenario 80 % Transformationspfad
+und 2,6 Mio. ha Anbauflächen im Jahr 2020 und 2050 der Tabelle 13 im
+Dokument
+["Technoökonomische Analyse und Transformationspfade des energetischen Biomassepotentials (TATBIO)"](../dbfz_biomass_heat_capacities/metadata.json)
+für die folgenden Konversionsanlagen von Biomasse entnommen:
+
+- Biomethan-Blockheizkraftwerk
+- Holzhackschnitzelkessel Sektor Industrie
+- Pelletkessel Sektor GHD
+- Holzhackschnitzelkessel Sektor GHD
+- Scheitholzvergaserkessel
+- Pelletkessel Sektor Gebäude
+- Biogasanlage + Blockheizkraftwerk
+- Biomethan Gas- und Dampfkombikraftwerk
+- Klärschlammfaulung + Blockheizkraftwerk
+- Papier-Zellstoff-KWK
+- Holzvergaser + Blockheizkraftwerk
+- Mikro-Holzgas-Blockheizkraftwerk
+
+Die Konversionstechnologien sind in der Spalte "technology" gelistet, während
+sich ihre installierten Leistungen für die beiden Projektionsjahre in den
+Spalten "capacity_[MW]_2020" und "capacity_[MW]_2050" befinden.
+
+In den Spalten "decentral" und "central" wird mit "x" angegeben, ob jeweils ein
+dezentraler und zentraler Einsatz der Konversionsanlage Stand der Technik ist.
+
+In der Spalte "carrier" wird analog zur Konvention der Namensgebung im
+Energiesystem (siehe [esys.md](../../../../docs/sections/esys.md)) der
+jeweilige in die Konversionsanlage eintretende Energieträger notiert.
+Diese werden Abbildung 3 des Dokuments entommen. Der Energieträger Schwarzlauge
+wird vereinfachend dem Energieträger feste Biomasse bzw. Holz zugeordnet.
+Klärgas und Holzgas werden vereinfachend Biogas zugeordnet.
+
+In der Spalte "tech" findet die Zuordnung zu der Technologie anhand der im
+Energiesystem verwendeten Komponenten (siehe
+[esys.md](../../../../docs/sections/esys.md)) statt.
diff --git a/digipipe/store/raw/dbfz_biomass_heat_capacities/metadata.json b/digipipe/store/raw/dbfz_biomass_heat_capacities/metadata.json
new file mode 100644
index 00000000..59f5bddb
--- /dev/null
+++ b/digipipe/store/raw/dbfz_biomass_heat_capacities/metadata.json
@@ -0,0 +1,95 @@
+{
+ "name": "dbfz_biomass_heat_capacities",
+ "title": "Technoökonomische Analyse und Transformationspfade des energetischen Biomassepotentials (TATBIO)",
+ "id": "dbfz_biomass_heat_capacities",
+ "description": "Installierte Leistungen von Biomasse-Konversionstechnologien. Die installierten Leistungen in MW wird im Szenario 80 % Transformationspfad",
+ "language": [
+ "de-DE"
+ ],
+ "subject": [],
+ "keywords": [
+ "Biomasse",
+ "Biomassepotential",
+ "Analyse",
+ "Transformationspfade",
+ "TATBIO"
+ ],
+ "publicationDate": "2019-05-08",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": ""
+ },
+ "temporal": {
+ "referenceDate": "2019-04-30",
+ "timeseries" : null
+ },
+ "sources": [
+ {
+ "title": "Technoökonomische Analyse und Transformationspfade des energetischen Biomassepotentials (TATBIO)",
+ "description": "Installierte Leistungen von Biomasse-Konversionstechnologien. Die installierten Leistungen in MW wird im Szenario 80 % Transformationspfad",
+ "path": "https://www.ufz.de/export/data/2/231891_technooekonomische-analyse-und-transformationspfade-des-energetischen-biomassepotentials(1).pdf",
+ "licenses": null
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-09-07",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": [],
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": [],
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+},
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/demandregio/data/.gitkeep b/digipipe/store/raw/demandregio/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/demandregio/dataset.md b/digipipe/store/raw/demandregio/dataset.md
new file mode 100644
index 00000000..108b85b4
--- /dev/null
+++ b/digipipe/store/raw/demandregio/dataset.md
@@ -0,0 +1,207 @@
+# DemandRegio
+
+Regionalisierte Bevölkerungsprognose, Haushalte sowie Strom- und Gasbedarfe
+inkl. Zeitreihen auf Landkreisebene.
+
+Die Daten wurden abgerufen mit einer
+[modifizierten Version des DemandRegio disaggregators](https://github.com/nesnoj/disaggregator),
+in der softwareseitige, jedoch keine methodischen Änderungen vorgenommen wurden.
+
+Der disaggregator basiert auf Daten bis 2017, anschließende Jahre werden
+fortgeschrieben.
+
+Weitere Informationen zum Projekt DemandRegio:
+
+- [Abschlussbericht](https://www.ffe.de/wp-content/uploads/2020/10/DemandRegio_Abschlussbericht.pdf)
+- [Abschlussworkshop](https://www.tu.berlin/er/forschung/projekte/demandregio-2)
+
+Die erzeugten Rohdaten wie unten beschrieben wurden mittels
+[API](http://opendata.ffe.de:4000/) abgerufen. Diese können alternativ direkt
+vom [OpenData-Portal der FfE](https://opendata.ffe.de/project/demandregio/)
+bezogen werden.
+
+Verwendetes Wetterjahr für Gasbedarfszeitreihen: 2011
+
+**Installation (in separater venv):**
+
+```commandline
+pip install disaggregator@git+https://github.com/nesnoj/disaggregator.git#egg=disaggregator
+```
+
+## Details zum Datenabruf
+
+### Bevölkerung
+
+Bevölkerung (Summe) und Bevölkerung je Haushaltsgröße (1, 2, 3, 4, 5, >5) je
+NUTS3.
+
+Jahre:
+
+- Bevölkerung bis 2017 historische Werte
+- Bevölkerung ab 2018 prognostizierte Werte basierend auf der 14. koordinierten
+ Bevölkerungsvorausberechnung der Statistischen Ämter von Bund und Ländern.
+- Haushalte nur 2011
+
+```python
+import pandas as pd
+from disaggregator import data
+
+# Population
+dr_hh_population = pd.DataFrame()
+for year in [2010, 2015, 2017, 2020, 2021, 2022, 2025, 2030, 2035, 2040, 2045]:
+ dr_hh_population[year] = round(data.population(year=year)).astype(int)
+
+dr_hh_population.to_csv("dr_hh_population.csv")
+
+# Households
+data.households_per_size().to_csv("dr_hh_households_2011.csv")
+```
+
+### Haushalte: Strom
+
+Bedarfe und SLP-Zeitreihen je NUTS3 mit Bottom-Up-Methode nach Haushaltsgröße.
+
+Jahre:
+
+- 2017: Letzte verfügbare Daten
+- 2022: Status quo, Fortschreibung mit Berücksichtigung Demografie und
+ Wanderung
+- 2035: Fortschreibungsjahr mit Berücksichtigung Demografie und Wanderung
+- 2045: Fortschreibungsjahr
+
+```python
+from disaggregator import spatial, temporal
+
+# Consumption
+spatial.disagg_households_power(
+ by="households",
+ weight_by_income=True,
+ year=2022,
+ scale_by_pop=True,
+).to_csv(f"dr_hh_power_demand_2022.csv")
+
+# Timeseries
+temporal.disagg_temporal_power_housholds_slp(
+ use_nuts3code=True,
+ by="households",
+ weight_by_income=True,
+ year=2022,
+ scale_by_pop=True,
+).to_csv(f"dr_hh_power_timeseries_2022.csv")
+```
+
+### Haushalte: Gas
+
+Zeitreihen je NUTS3
+
+```python
+from disaggregator import temporal
+
+# Timeseries
+temporal.disagg_temporal_gas_households(
+ use_nuts3code=True,
+ how='top-down',
+ year=2011,
+).to_csv(f"dr_hh_gas_timeseries_2011.csv")
+```
+
+
+### GHD und Industrie: Strom
+
+Bedarfe und Zeitreihen je NUTS3:
+
+- Bedarfe: Je Wirtschaftszweig (WZ), abzüglich Eigenerzeugung
+- Zeitreihen: Für alle WZ bedarfsgewichtet aggregiert, Einzelprofile basieren
+ je nach WZ auf gemessenen oder SLP inkl. Wanderung
+- Letzte verfügbare Daten aus 2017, Fortschreibung für 2022 mit
+ Berücksichtigung Beschäftigte und Effizienzgewinne
+
+```python
+from disaggregator import spatial, temporal
+
+#######
+# CTS #
+#######
+
+# Consumption
+spatial.disagg_CTS_industry(
+ sector='CTS',
+ source='power',
+ use_nuts3code=True,
+ year=2022,
+).to_csv("dr_cts_power_demand_2022.csv")
+# Timeseries
+temporal.disagg_temporal_power_CTS(
+ detailed=False,
+ use_nuts3code=True,
+ year=2022,
+).to_csv("dr_cts_power_timeseries_2022.csv")
+
+############
+# Industry #
+############
+
+# Consumption
+spatial.disagg_CTS_industry(
+ sector='industry',
+ source='power',
+ use_nuts3code=True,
+ year=2022,
+).to_csv("dr_ind_power_demand_2022.csv")
+# Timeseries
+temporal.disagg_temporal_industry(
+ source="power",
+ detailed=False,
+ use_nuts3code=True,
+ no_self_gen=False,
+ year=2022,
+).to_csv("dr_ind_power_timeseries_2022.csv")
+```
+
+### GHD: Gas
+
+Zeitreihen je NUTS3 für alle WZ bedarfsgewichtet aggregiert, Einzelprofile
+basieren je nach WZ auf gemessenen oder SLP inkl. Wanderung. Letzte verfügbare
+Daten aus 2017, Fortschreibung für 2022 mit Berücksichtigung Beschäftigte und
+Effizienzgewinne.
+
+```python
+from disaggregator import spatial, temporal
+
+# Timeseries
+x=temporal.disagg_temporal_gas_CTS(
+ detailed=False,
+ use_nuts3code=True,
+ year=2011,
+).to_csv("dr_cts_gas_timeseries_2011.csv")
+```
+
+### Industrie: Gas
+
+Bedarfe und Zeitreihen je NUTS3:
+
+- Bedarfe: Je Wirtschaftszweig (WZ), abzüglich Eigenerzeugung
+- Zeitreihen: Für alle WZ bedarfsgewichtet aggregiert, Einzelprofile basieren
+ je nach WZ auf gemessenen oder SLP inkl. Wanderung
+- Letzte verfügbare Daten aus 2017, Fortschreibung für 2022 mit
+ Berücksichtigung Beschäftigte und Effizienzgewinne
+
+```python
+from disaggregator import spatial, temporal
+
+# Consumption
+spatial.disagg_CTS_industry(
+ sector='industry',
+ source='gas',
+ use_nuts3code=True,
+ year=2022,
+).to_csv("dr_ind_gas_demand_2022.csv")
+# Timeseries
+x=temporal.disagg_temporal_industry(
+ source="gas",
+ detailed=False,
+ use_nuts3code=True,
+ no_self_gen=False,
+ year=2011,
+).to_csv("dr_ind_gas_timeseries_2011.csv")
+```
diff --git a/digipipe/store/raw/demandregio/metadata.json b/digipipe/store/raw/demandregio/metadata.json
new file mode 100644
index 00000000..628e4194
--- /dev/null
+++ b/digipipe/store/raw/demandregio/metadata.json
@@ -0,0 +1,150 @@
+{
+ "name": "demandregio",
+ "title": "DemandRegio",
+ "id": "demandregio",
+ "description": "Regionalisierte Bevölkerungsprognose, Haushalte sowie Strom- und Gasbedarfe inkl. Zeitreihen auf Landkreisebene.",
+ "language": [
+ "en-GB",
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "Bevölkerung",
+ "Bevölkerungsprognose",
+ "Strombedarf",
+ "Gasbedarf",
+ "Haushalte",
+ "disaggregator",
+ "disaggregation"
+ ],
+ "publicationDate": "2020-09-30",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": "NUTS-3"
+ },
+ "temporal": {
+ "referenceDate": "2022-01-01",
+ "timeseries": [
+ {
+ "start": "2022-01-01T00:00+01",
+ "end": "2022-12-31T23:00+01",
+ "resolution": "1 h",
+ "alignment": null,
+ "aggregationType": "sum"
+ },
+ {
+ "start": "2035-01-01T00:00+01",
+ "end": "2035-12-31T23:00+01",
+ "resolution": "1 h",
+ "alignment": null,
+ "aggregationType": "sum"
+ },
+ {
+ "start": "2045-01-01T00:00+01",
+ "end": "2045-12-31T23:00+01",
+ "resolution": "1 h",
+ "alignment": null,
+ "aggregationType": "sum"
+ },
+ {
+ "start": "2011-01-01T00:00+01",
+ "end": "2011-12-31T23:00+01",
+ "resolution": "1 h",
+ "alignment": null,
+ "aggregationType": "sum"
+ },
+ {
+ "start": "2022-01-01T00:00+01",
+ "end": "2022-12-31T23:00+01",
+ "resolution": "1 h",
+ "alignment": null,
+ "aggregationType": "sum"
+ }
+ ]
+ },
+"sources": [
+ {
+ "title": "DemandRegio",
+ "description": "Regionalisierte Bevölkerungsprognose, Haushalte sowie Strom- und Gasbedarfe inkl. Zeitreihen auf Landkreisebene.",
+ "path": "https://github.com/nesnoj/disaggregator/",
+ "licenses": [
+ {
+ "name": "CC BY 4.0 DE",
+ "title": "Creative Commons Namensnennung 4.0 Deutschland",
+ "path": "https://creativecommons.org/licenses/by/4.0/deed.de",
+ "instruction": "Sie müssen angemessene Urheber- und Rechteangaben machen, einen Link zur Lizenz beifügen und angeben, ob Änderungen vorgenommen wurden. Diese Angaben dürfen in jeder angemessenen Art und Weise gemacht werden, allerdings nicht so, dass der Eindruck entsteht, der Lizenzgeber unterstütze gerade Sie oder Ihre Nutzung besonders.",
+ "attribution": "© FZJ, TUB, FfE"
+ },
+ {
+ "name": "GNU FDL" ,
+ "title": "GNU General Public License v3.0",
+ "path": "https://www.gnu.org/licenses/gpl-3.0.en.html",
+ "instruction": "Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed",
+ "attribution": "© FZJ, TUB, FfE"
+ }
+ ]
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-08-15",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": [],
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/destatis_gv/data/.gitkeep b/digipipe/store/raw/destatis_gv/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/destatis_gv/dataset.md b/digipipe/store/raw/destatis_gv/dataset.md
new file mode 100644
index 00000000..438fd699
--- /dev/null
+++ b/digipipe/store/raw/destatis_gv/dataset.md
@@ -0,0 +1,3 @@
+# Bevölkerung
+
+Einwohnerzahl nach Gemeinden des Statistischen Bundesamts.
diff --git a/digipipe/store/raw/destatis_gv/metadata.json b/digipipe/store/raw/destatis_gv/metadata.json
new file mode 100644
index 00000000..22e76b0c
--- /dev/null
+++ b/digipipe/store/raw/destatis_gv/metadata.json
@@ -0,0 +1,110 @@
+{
+ "name": "destatis_gv",
+ "title": "Adminstratives Gemeinndeverzeichnis",
+ "id": "destatis_gv",
+ "description": "Alle politisch selbständigen Gemeinden mit ausgewählten Merkmalen am 31.12.2022 ",
+ "language": [
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "destatis",
+ "gemeindeverzeichnis"
+ ],
+ "publicationDate": "2023-01-12",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": null,
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": ""
+ },
+ "temporal": {
+ "referenceDate": "2022-02-14",
+ "timeseries": null
+ },
+ "sources": [
+ {
+ "title": "Statistisches Bundesamt",
+ "description": "Alle politisch selbständigen Gemeineden mit ausgewählten Merkmalen am 31.12.2022 (4.Quartal)",
+ "path": "https://www.destatis.de/DE/Themen/Laender-Regionen/Regionales/Gemeindeverzeichnis/Administrativ/Archiv/GVAuszugQ/AuszugGV4QAktuell.html",
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Statistisches Bundesamt (Destatis), 2023"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Statistisches Bundesamt (Destatis), 2023"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "hedwiglieselotte",
+ "email": "hedwig.bartels@rl-institut.de",
+ "date": "2023-03-28",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": null,
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/dwd_temperature/data/.gitkeep b/digipipe/store/raw/dwd_temperature/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/dwd_temperature/dataset.md b/digipipe/store/raw/dwd_temperature/dataset.md
new file mode 100644
index 00000000..f1dbd8df
--- /dev/null
+++ b/digipipe/store/raw/dwd_temperature/dataset.md
@@ -0,0 +1,23 @@
+# Temperatur
+
+Stündliche Mittelwerte der Luft- und Erdbodentemperatur des Deutschen
+Wetterdienstes
+([Climate Data Center](https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/hourly/))
+für das Jahr 2011 je Gemeinde in der Region ABW, vorverarbeitet im Projekt
+[WindNODE](https://windnode-abw.readthedocs.io/en/latest/energy_system_model.html#energy-demand-today).
+
+Werte:
+
+- `temp_amb`: Lufttemperatur in 2 m Höhe
+- `temp_soil`: Erdbodentemperatur in 1 m Tiefe
+
+Verwendete Stationen:
+
+- Wittenberg
+- Köthen
+- Jessnitz
+- Seehausen
+- Holzdorf
+
+Die Zuordnung der Stationsmesswerte zu Gemeinden erfolgte über die jeweils
+nächstgelegene Wetterstation.
diff --git a/digipipe/store/raw/dwd_temperature/metadata.json b/digipipe/store/raw/dwd_temperature/metadata.json
new file mode 100644
index 00000000..f358c0e9
--- /dev/null
+++ b/digipipe/store/raw/dwd_temperature/metadata.json
@@ -0,0 +1,118 @@
+{
+ "name": "dwd_temperature",
+ "title": "temperatur_2011",
+ "id": "dwd_temperature",
+ "description": "Stündliche Mittelwerte der Luft- und Erdbodentemperatur des Deutschen Wetterdienstes (Climate Data Center) für das Jahr 2011 je Gemeinde in der Region ABW, vorverarbeitet im Projekt WindNODE.",
+ "language": [
+ "en-GB"
+ ],
+ "subject": null,
+ "keywords": [
+ "Wetter",
+ "Temperatur"
+ ],
+ "publicationDate": null,
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Anhalt-Bitterfeld-Wittenberg",
+ "extent": "Germany",
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": "2023-05-23",
+ "timeseries": [
+ {
+ "start": "2011-01-01T00:00+01",
+ "end": "2011-12-31T23:00+01",
+ "resolution": "1 h",
+ "alignment": "left",
+ "aggregationType": null
+ }
+ ]
+ },
+"sources": [
+ {
+ "title": "temperatur_2011",
+ "description": "Stündliche Mittelwerte der Luft- und Erdbodentemperatur des Deutschen Wetterdienstes (Climate Data Center) für das Jahr 2011 je Gemeinde in der Region ABW, vorverarbeitet im Projekt WindNODE",
+ "path": "https://www.dwd.de/DE/leistungen/cdc/climate-data-center.html",
+ "licenses": [
+ {
+ "name": "GeoNutzV",
+ "title": "Verordnung zur Festlegung der Nutzungsbestimmungen für die Bereitstellung von Geodaten des Bundes",
+ "path": "https://www.gesetze-im-internet.de/geonutzv/GeoNutzV.pdf",
+ "instruction": "Alle frei zugänglichen Geodaten und Geodatendienste dürfen entsprechend der Verordnung zur Festlegung der Nutzungsbestimmungen für die Bereitstellung von Geodaten des Bundes (GeoNutzV) unter Beigabe eines Quellenvermerks ohne Einschränkungen weiterverwendet werden.",
+ "attribution": " © Deutscher Wetterdienst"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": "GeoNutzV",
+ "title": "Verordnung zur Festlegung der Nutzungsbestimmungen für die Bereitstellung von Geodaten des Bundes",
+ "path": "https://www.gesetze-im-internet.de/geonutzv/GeoNutzV.pdf",
+ "instruction": "Alle frei zugänglichen Geodaten und Geodatendienste dürfen entsprechend der Verordnung zur Festlegung der Nutzungsbestimmungen für die Bereitstellung von Geodaten des Bundes (GeoNutzV) unter Beigabe eines Quellenvermerks ohne Einschränkungen weiterverwendet werden.",
+ "attribution": " © Deutscher Wetterdienst"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "aaron.schilling@rl-institut.de",
+ "date": "2023-08-25",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": [],
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/emissions/data/.gitkeep b/digipipe/store/raw/emissions/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/emissions/dataset.md b/digipipe/store/raw/emissions/dataset.md
new file mode 100644
index 00000000..ca296b17
--- /dev/null
+++ b/digipipe/store/raw/emissions/dataset.md
@@ -0,0 +1,238 @@
+# Emissionen
+
+Emissionen für die Jahre 1990 und 2019 für Sachsen-Anhalt und disaggregiert für
+die Region Anhalt-Bitterfeld-Wittenberg (ABW). Die Grundlage hierfür ist der
+[THG-Bericht 2021](https://lau.sachsen-anhalt.de/fileadmin/Bibliothek/Politik_und_Verwaltung/MLU/LAU/Wir_ueber_uns/Publikationen/Fachberichte/Dateien/221014_THG-Bericht.pdf)
+Sachsen-Anhalt (ST).
+
+Datei: `emissions.csv`, Felder:
+
+- `sector`: Sektor
+- `cat`: Kategorie ("*" = alle)
+- `subcat`: Unterkategorie ("*" = alle)
+- `name`: Bezeichner
+- `st`: Emissionen Sachsen-Anhalt in kt CO2-Äquivalent
+- `abw`: Emissionen Region ABW in kt CO2-Äquivalent
+
+`sector`, `cat` und `subcat` folgen der Nomenklatur des Common Reporting Formats
+(CRF) nach [KSG Anlage 1](https://www.gesetze-im-internet.de/ksg/anlage_1.html).
+[Grafik hierzu](https://expertenrat-klima.de/content/uploads/2023/05/ERK2023_Pruefbericht-Emissionsdaten-des-Jahres-2022.pdf)
+(Abb. 2 auf S. 30).
+
+## Disaggregation
+
+Anhand unterschiedlicher Kriterien und Datenquellen wurde näherungsweise von den
+vorliegenden Emissionen für Sachsen-Anhalt für 1990 und 2019 auf die Region ABW
+disaggregiert. Je Sektor sind hier die gewählten
+**energiebestimmenden Größen (EnbG)** angegeben, sowie die Herangehensweise zur
+jeweiligen Berechnung.
+
+### Sektor Energiewirtschaft (CRF 1.A.1 + 1.B)
+
+Aus der Liste der
+[Emissionshandelspflichtigen Anlagen](https://www.dehst.de/SharedDocs/downloads/DE/anlagenlisten/2013-2020/2020.pdf?__blob=publicationFile&v=3)
+wurden jene Daten zu Anlagen extrahiert, welche sich in Sachsen-Anhalt befinden
+und als Bezeichnung "Energieumwandlung >= 50 MW FWL" oder "Energieumwandlung
+20–50 MW FWL" (Haupttätigkeit nach TEHG) aufweisen.
+Die Summe der angegebenen Emissionen (t CO2 Äq) jener Anlagen, welche in der
+Region ABW liegen, wurde in Relation zu der Summe der Emissionen aus den Anlagen
+in Gesamt-ST gesetzt. Dieser Anteil wurde auf die im THG-Bericht angegebene
+Emissionsmenge im Sektor "Energiewirtschaft (1.A.1)" sowie "Prozessemissionen
+(1.B)" angelegt und so für ABW näherungsweise disaggregiert.
+
+Hinweise:
+
+- Aufgrund mangelnder Daten wurde für das Jahr 1990 auf die neuesten verfügbaren
+ Daten (2005-2007) aus der Anlagenliste zurückgegriffen.
+- Energiewirtschaftlich relevante Anlagen unter 20 MW FWL sind in der
+ Anlagenliste nicht erfasst und konnten somit nicht berücksichtigt werden.
+
+Quellen:
+
+- [Emissionshandelspflichtige Anlagen in Deutschland 2020 (Stand 03.05.2021)](https://www.dehst.de/SharedDocs/downloads/DE/anlagenlisten/2013-2020/2020.pdf?__blob=publicationFile&v=3)
+- [Treibhausgasemissionen in Sachsen-Anhalt 2018 (Stand 12.05.2021)](https://lau.sachsen-anhalt.de/fileadmin/Bibliothek/Politik_und_Verwaltung/MLU/LAU/Wir_ueber_uns/Publikationen/Fachberichte/Dateien/THG_Bericht_2018.pdf)
+
+#### CRF 1.A.1
+
+Energiewirtschaft (Umwandlungsbereich): umfasst die öffentliche Elektrizitäts-
+und Wärmeversorgung sowie Raffinerien.
+
+EnbG: Emissionen aus europäischem Emissionshandel
+
+#### CRF 1.B
+
+Diffuse Emissionen aus Brennstoffen: Diese Kategorie beinhaltet flüchtige
+Emissionen aus der Gewinnung, Verarbeitung und Verteilung von Brennstoffen. Die
+wichtigsten Quellen sind die Verteilung von Erdgas, aber auch Emissionen aus
+Förderung und Abfackelung, die Extraktion und Umwandlung von Braunkohle,
+Emissionen aus der Raffination von Erdöl sowie Emissionen aus der Lagerung und
+Verteilung von Mineralölprodukten.
+
+EnbG: Emissionen aus europäischem Emissionshandel
+
+### Sektor Industrie (CRF 1.A.2)
+
+Dieser Sektor umfasst sämtliche energiebedingten Emissionen durch verarbeitendes
+Gewerbe.
+
+Zur Disaggregierung wurde der Energieverbrauch der Industriebetriebe in ABW mit
+dem Gesamtenergieverbrauch aller Industriebetriebe in Sachsen-Anhalt in Relation
+gesetzt. Dabei wurde eine Differenzierung hinsichtlich der
+Energieträgerzusammensetzung von ABW im Vergleich zu ST durchgeführt und anhand
+von Emissionsfaktoren berechnet.
+
+EnbG: Energieverbrauch nach Energieträgern
+
+Quellen:
+
+- [Energieverbrauch der Industriebetriebe in Sachsen-Anhalt nach ausgewählten Energieträgern und Kreisen](https://statistik.sachsen-anhalt.de/fileadmin/Bibliothek/Landesaemter/StaLa/startseite/Themen/Energie/Tabellen/Energieverwendung/Energieverbrauch_nach_Kreisen_ab_dem_Jahr_2010.xlsx)
+- [Emissionsfaktor für Stromerzeugung (UBA)](https://www.umweltbundesamt.de/sites/default/files/medien/479/bilder/dateien/entwicklung_der_spezifischen_emissionen_des_deutschen_strommix_1990-2020_und_erste_schaetzungen_2021.pdf)
+- [BISKO Bilanzierungs-Systematik Kommunal (Aktualisierung 11/2019)](https://www.ifeu.de/fileadmin/uploads/BISKO_Methodenpapier_kurz_ifeu_Nov19.pdf)
+
+### Sektor Prozessemissionen (CRF 2)
+
+Dieser Sektor umfasst sämtliche Emissionen, welche durch Industrieprozesse
+anfallen. Dies sind Emissionen aus: Herstellung mineralischer Produkte,
+chemischer Industrie, Herstellung von Metallen, übrigen Prozessen und
+Produktverwendungen (CRF 2.A-H).
+Zur Disaggregierung wurde erneut die
+[Liste der Emissionshandelspflichtigen Anlagen](https://www.dehst.de/SharedDocs/downloads/DE/anlagenlisten/2013-2020/2020.pdf?__blob=publicationFile&v=3)
+herangezogen. Anders als im Sektor Energiewirtschaft (s.o.) wurde jedoch der
+Anteil aller Anlagen, welche nicht der Energiewirtschaft zugerechnet werden, zur
+Bestimmung des Anteils von ABW an ST gewählt.
+
+EnbG: Emissionen aus europäischem Emissionshandel
+
+### Sektor Verkehr (CRF 1.A.3)
+
+Dieser Sektor umfasst Emissionen aus dem Straßenverkehr, dem zivilen
+Luftverkehr, aus dem Schiffsverkehr, verbrennungsbedingte Emissionen aus dem
+Schienenverkehr sowie Emissionen des übrigen Verkehrs und weitere Quellen zur
+Bereitstellung der im Verkehr verbrauchten Energie. Die Verbrennung von
+Mineralölprodukten im Straßenverkehr spielt die größte Rolle und macht weit über
+90 % der sektoralen Emissionen aus. Daher wird zur Disaggreagation der
+motorisierte Straßenverkehr über zugelassene Kraftfahrzeuge mit
+durchschnittlichen Fahrleistungen und spezifischer Emissionen pro Kilometer und
+Fahrzeugklasse herangezogen.
+
+Hierfür wird zunächst aus
+[Verkehr in Kilometern (VK) ZeitreiheJahre 2014 - 2022](https://www.kba.de/DE/Statistik/Kraftverkehr/VerkehrKilometer/vk_inlaenderfahrleistung/vk_inlaenderfahrleistung_node.html;jsessionid=DD419FD0604C0BCC72A9E4533BB0319F.live21324)
+und
+[Umweltfreundlich mobil! Ein ökologischer Verkehrsartenvergleich für den Personen- und Güterverkehr in Deutschland)](https://www.umweltbundesamt.de/sites/default/files/medien/5750/publikationen/2021_fb_umweltfreundlich_mobil_bf.pdf)
+ein durchschnittlicher Emissionswert pro Jahr und Fahrzeugklasse ermittelt.
+Dieser wird mit den zugelassenen Fahrzeugen der entsprechenden Fahrzeugklassen
+aus
+[Kraftfahrzeugbestand nach Kraftfahrzeugarten - Stichtag 01.01. - regionale Tiefe: Kreise und krfr. Städte (bis 01.01.2019)](https://www-genesis.destatis.de/genesis//online?operation=table&code=46251-0001&bypass=true&levelindex=0&levelid=1691405772899#abreadcrumb)
+einerseits für ganz Sachsen-Anhalt und andererseits ABW multipliziert. Daraus
+wird ein Verhältnis der Verkehrsemissionen in ABW zu ST gewonnen.
+
+Hinweise:
+
+- Die Datenlage für die zugelassenen Fahrzeuge, gefahrenen Kilometer und
+ Emissionen pro km sind nicht spezifisch für 1990 sondern nur für einzelne
+ Jahre der frühen 1990er verfügbar. Daher ist der Emissionswert für 1990 mit
+ einer höheren Unsicherheit behaftet.
+
+EnbG:
+
+- Zugelassene Kraftfahrzeuge
+- Durchschnittliche Fahrleistung und spez. CO2 Emission pro km und
+ Fahrzeugklasse
+
+Quellen:
+
+- [Kraftfahrzeugbestand nach Kraftfahrzeugarten - Stichtag 01.01. - regionale Tiefe: Kreise und krfr. Städte (bis 01.01.2019)](https://www-genesis.destatis.de/genesis//online?operation=table&code=46251-0001&bypass=true&levelindex=0&levelid=1691405772899#abreadcrumb)
+- [Umweltfreundlich mobil! Ein ökologischer Verkehrsartenvergleich für den Personen- und Güterverkehr in Deutschland)](https://www.umweltbundesamt.de/sites/default/files/medien/5750/publikationen/2021_fb_umweltfreundlich_mobil_bf.pdf)
+- [Verkehr in Kilometern (VK) ZeitreiheJahre 2014 - 2022](https://www.kba.de/DE/Statistik/Kraftverkehr/VerkehrKilometer/vk_inlaenderfahrleistung/vk_inlaenderfahrleistung_node.html;jsessionid=DD419FD0604C0BCC72A9E4533BB0319F.live21324)
+
+### Sektor Sonstige Energie (insbes. Gebäude) (CRF 1.A.4 + 1.A.5)
+
+Dieser Sektor umfasst den durch Energieumwandlung nicht bereits abgedeckten
+Energiebedarf. Das sind vor allem kleine Einzelfeuerungsanlagen bis hin zu
+immissionsschutzrechtlich genehmigungsbedürftigen Anlagen mit einer
+Nennwärmeleistung von mehreren Megawatt. Zur Disaggreagtion wurde daher der
+Wärmebedarf von ABW im Verhältnis zum Wärmebedarf von gesamt Sachsen Anhalt
+gewählt. Der Wärmevedarf umfasst Raumwärme, Warmwasser sowie Kochen und wird aus
+Daten aus dem Pipeline-Datensatz
+[demand_heat_region](../../datasets/demand_heat_region/dataset.md) generiert.
+
+Ergebnis: 17,46 % des Bedarfs in Sachsen-Anhalt entfällt auf ABW.
+
+Code
+```
+# Sektor HH
+heat_hh_dist_states = gpd.read_file("demand_heat_zonal_stats-res-bkg_vg250_federal_states.gpkg")
+heat_hh_demand_st = float(heat_hh_dist_states.loc[heat_hh_dist_states.nuts == "DEE"].heat_demand)
+heat_hh_demand_abw = gpd.read_file("demand_heat_zonal_stats-res-bkg_vg250_muns_region.gpkg").heat_demand.sum()
+
+# Sektor GHD
+heat_cts_dist_states = gpd.read_file("demand_heat_zonal_stats-ser-bkg_vg250_federal_states.gpkg")
+heat_cts_demand_st = float(heat_cts_dist_states.loc[heat_cts_dist_states.nuts == "DEE"].heat_demand)
+heat_cts_demand_abw = gpd.read_file("demand_heat_zonal_stats-ser-bkg_vg250_muns_region.gpkg").heat_demand.sum()
+
+# Anteil ABW an ST
+heat_share = (heat_hh_demand_abw + heat_cts_demand_abw) / (heat_hh_demand_st + heat_cts_demand_st)
+```
+
+EnbG: Wärmebedarf aus Energiesystem
+
+### Sektor Landwirtschaft (CRF 3)
+
+Der Sektor umfasst Emissionen aus der Viehwirtschaft und der Bewirtschaftung von
+Böden. Daher werden zunächst die Emissionsunterkategorien 3.A-J der
+Viehwirtschaft oder der Bewirtschaftung von Böden zugeordnet. Anschließend
+werden diese getrennt nach den Viehbeständen bzw. der landwirtschaftlich
+genutzen Fläche disaggreiert.
+
+#### CRF 3.A - Landwirtschaft – Fermentation
+
+Emissionen durch Fermentation (CRF 3.A) entstehen vorrangig durch
+Verdauungsprozesse in der Viehwirtschaft. Deswegen wird der Anteil ABWs an
+diesen Emissionen durch die Viehbestände abgeschätzt.
+
+Hinweis:
+
+- Die Viehbestände für 1990 sind nicht bekannt, es wird stattdessen auf die
+ Viehbestände von 1996 zurückggegriffen.
+
+EnbG: Viehbestände
+
+Quelle:
+
+- [Viehbestand der landwirtschaftlichen Betriebe in Großvieheinheiten (GV) nach Jahren und Kreisen)](https://statistik.sachsen-anhalt.de/themen/wirtschaftsbereiche/land-und-forstwirtschaft-fischerei/tabellen-viehwirtschaft-und-tierische-erzeugnisse#c234218)
+
+##### CRF 3.B-J
+
+In den Unterkategorien 3.C-J ist eine Proportionalität der Emissionen und der
+landwirtschafltich genutzen Fläche zu erwarten. Unterkategorie 2.B
+"Wirtschaftsdüngerausbringung (ohne Gärreste)" ist allerdings ein Grenzfall, da
+er aus Abfällen der Tierhaltung produziert wird und bereits hierbei
+Treibhausgase entstehen, diese aber nicht vor Ort eingesetzt werden müssen,
+sondern auf beliebigen landwirtschafltichen Flächen eingesetzt werden kann.
+Daher wird hier auch diese Unterkategorie der Landnutzung zugeordnet.
+
+Hinweis:
+
+- die Flächenntuzungsdaten gehen nicht bis 1990 zurück, ändern sich über die
+ Jahre aber nur marginal, sodass hier nur von geringen Abweichungen auszugehen
+ ist.
+
+EnbG: Landwirtschaftlich genutzte Fläche
+
+Quelle:
+
+- [Flaeche_nach_Kultuarten_nach_Jahren_und_Kreisen](https://statistik.sachsen-anhalt.de/themen/wirtschaftsbereiche/land-und-forstwirtschaft-fischerei/tabellen-bodennutzung-und-anbau)
+
+### Sektor Abfall und Abwasser (CRF 5)
+
+Dieser Sektor besteht vor allem aus Emissionen aus Abfalldeponien, welche der
+Zersetzung organischer Materialien in Deponien entstehen. Es wird angenommen,
+dass der Abfall aus Produktionsprozessen gegenüber den Abfällen aus Konsum
+vernachlässigbar sind, weswegen eine Disaggregation auf Grundlage der
+Bevölkerung von ABW vorgenommen wird.
+
+EnbG: Bevölkerung
+
+Quelle:
+
+- [Bevölkerung nach Geschlecht in den Gemeinden](https://genesis.sachsen-anhalt.de/genesis//online?operation=table&code=12411-0001&bypass=true&levelindex=0&levelid=1691507280245#abreadcrumb)
diff --git a/digipipe/store/raw/emissions/metadata.json b/digipipe/store/raw/emissions/metadata.json
new file mode 100644
index 00000000..9afd4d47
--- /dev/null
+++ b/digipipe/store/raw/emissions/metadata.json
@@ -0,0 +1,6 @@
+{
+ "Daten Sachsen-Anhalt": "https://lau.sachsen-anhalt.de/fileadmin/Bibliothek/Politik_und_Verwaltung/MLU/LAU/Wir_ueber_uns/Publikationen/Fachberichte/Dateien/221014_THG-Bericht.pdf",
+ "Datensätze Desaggregation": {
+ "Industrie": ""
+ }
+}
diff --git a/digipipe/store/raw/eurostat_lau/data/.gitkeep b/digipipe/store/raw/eurostat_lau/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/eurostat_lau/dataset.md b/digipipe/store/raw/eurostat_lau/dataset.md
new file mode 100644
index 00000000..8e6f679d
--- /dev/null
+++ b/digipipe/store/raw/eurostat_lau/dataset.md
@@ -0,0 +1,5 @@
+# Lokale Verwaltungseinheiten
+
+Lokale Verwaltungseinheiten (LAUs) von Eurostat, mit NUTS kompatibel. Diese LAU
+sind die Bausteine der NUTS und umfassen die Gemeinden und Kommunen der
+Europäischen Union.
diff --git a/digipipe/store/raw/eurostat_lau/metadata.json b/digipipe/store/raw/eurostat_lau/metadata.json
new file mode 100644
index 00000000..d29a7266
--- /dev/null
+++ b/digipipe/store/raw/eurostat_lau/metadata.json
@@ -0,0 +1,111 @@
+{
+ "name": "eurostat_lau",
+ "title": "Lokale Verwaltungseinheiten",
+ "id": "eurostat_lau",
+ "description": "Lokale Verwaltungseinheiten (LAUs) von Eurostat, mit NUTS kompatibel. Diese LAU sind die Bausteine der NUTS und umfassen die Gemeinden und Kommunen der Europäischen Union",
+ "language": [
+ "en-GB"
+ ],
+ "subject": null,
+ "keywords": [
+ "Verwaltungseinheiten",
+ "NUTS",
+ "LAU"
+ ],
+ "publicationDate": "2022-12-15",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": null,
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": "NUTS-3"
+ },
+ "temporal": {
+ "referenceDate": "2022-06-30",
+ "timeseries": null
+ },
+ "sources": [
+ {
+ "title": "Lokale Verwaltungseinheiten",
+ "description": "Lokale Verwaltungseinheiten (LAUs) von Eurostat, mit NUTS kompatibel. Diese LAU sind die Bausteine der NUTS und umfassen die Gemeinden und Kommunen der Europäischen Union",
+ "path": "https://ec.europa.eu/eurostat/de/web/nuts/local-administrative-units",
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© eurostat, 2023"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© eurostat, 2023"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-08-25",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": null,
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/osm/data/.gitkeep b/digipipe/store/raw/osm/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/osm/dataset.md b/digipipe/store/raw/osm/dataset.md
new file mode 100644
index 00000000..e5179f87
--- /dev/null
+++ b/digipipe/store/raw/osm/dataset.md
@@ -0,0 +1,7 @@
+# OpenStreetMap
+
+OpenStreetMap Datenauszug Deutschland.
+
+Quelle: https://download.geofabrik.de/europe/germany-230101.osm.pbf
+
+Ist nicht Teil des Eingangsdaten-Packages - manueller Download erforderlich.
diff --git a/digipipe/store/raw/osm/metadata.json b/digipipe/store/raw/osm/metadata.json
new file mode 100644
index 00000000..aa93741c
--- /dev/null
+++ b/digipipe/store/raw/osm/metadata.json
@@ -0,0 +1,111 @@
+{
+ "name": "openstreetmap",
+ "title": "",
+ "id": "openstreetmap",
+ "description": "OpenStreetMap extract",
+ "language": [
+ "de-DE",
+ "en-GB"
+ ],
+ "subject": [],
+ "keywords": [
+ "openstreetmap",
+ "osm"
+ ],
+ "publicationDate": "2023-06-30",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": ""
+ },
+ "temporal": {
+ "referenceDate": "2023-06-30",
+ "timeseries": []
+ },
+ "sources": [
+ {
+ "title": "OpenStreetMap Data Extracts (Geofabrik)",
+ "description": "Full data extract of OpenStreetMap data",
+ "path": "https://download.geofabrik.de/europe/germany-230101.osm.pbf",
+ "licenses": [
+ {
+ "name": "ODbL-1.0",
+ "title": "Open Data Commons Open Database License 1.0",
+ "path": "https://opendatacommons.org/licenses/odbl/1.0/",
+ "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!",
+ "attribution": "© OpenStreetMap contributors"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": "ODbL-1.0",
+ "title": "Open Data Commons Open Database License 1.0",
+ "path": "https://opendatacommons.org/licenses/odbl/1.0/",
+ "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!",
+ "attribution": "© OpenStreetMap contributors"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "nesnoj",
+ "email": "jonathan.amme@rl-institut.de",
+ "date": "2023-06-30",
+ "object": "metadata",
+ "comment": "Create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": [],
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "oemetadata_v1.5.1",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/regiostat/data/.gitkeep b/digipipe/store/raw/regiostat/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/regiostat/dataset.md b/digipipe/store/raw/regiostat/dataset.md
new file mode 100644
index 00000000..89b6be58
--- /dev/null
+++ b/digipipe/store/raw/regiostat/dataset.md
@@ -0,0 +1,80 @@
+# Regionalstatistik (GENESIS)
+
+Enthält folgende Datensätze der statistischen Ämter des Bundes und der Länder:
+
+## Energieverwendung der Betriebe im Verarbeitenden Gewerbe (43531-01-02-4)
+
+Jahreserhebung ü. die Energieverwendung der Betriebe im verarbeitendem Gewerbe.
+
+Der Datensatz umfasst:
+
+- Betriebe des Verarbeitenden Gewerbes sowie des Bergbaus
+und der Gewinnung von Steinen und Erden von Unternehmen des
+Produzierenden Gewerbes mit im Allgemeinen 20 und mehr
+Beschäftigten.
+- Betriebe des Verarbeitenden Gewerbes sowie des Bergbaus
+und der Gewinnung von Steinen und Erden mit im Allgemeinen
+20 und mehr Beschäftigten von Unternehmen der übrigen
+Wirtschaftsbereiche.
+Die Berichterstattung schließt Verarbeitende Betriebe des
+Handwerks ein.
+Bei 7 Wirtschaftszweigen gilt eine Abschneidegrenze von 10
+Beschäftigten. Die Merkmalswerte beziehen sich auf den
+gesamten Betrieb, schließen damit die nicht produzierenden
+Betriebsteile mit ein.
+Maßgebend für die Zuordnung ist ab 2008 die „Klassifikation
+der Wirtschaftszweige, Ausgabe 2008 (WZ 2008)“, und zwar
+die Abschnitte B und C.
+
+- Datei: `43531-01-02-4.xlsx`
+- Stand: 2021
+
+## Betriebe, tätige Personen, Bruttoentgelte (42111-01-04-5)
+
+Jahreserhebung ü. Betriebe, tätige Personen und Bruttoentgelte der Betriebe im
+verarbeitendem Gewerbe.
+
+Der Datensatz umfasst:
+
+- Sämtliche Betriebe des Wirtschaftsbereiches Verarbeitendes
+Gewerbe sowie Bergbau und Gewinnung von Steinen und Erden,
+wenn diese Betriebe zu Unternehmen des Bereiches
+Verarbeitendes Gewerbe sowie Bergbau und Gewinnung von
+Steinen und Erden gehören und in diesen Unternehmen
+mindestens 20 Personen tätig sind;
+- die Betriebe des Wirtschaftsbereiches Verarbeitendes
+Gewerbe sowie Bergbau und Gewinnung von Steinen und Erden
+mit mindestens 20 tätigen Personen, sofern diese Betriebe
+zu Unternehmen gehören, deren wirtschaftlicher Schwerpunkt
+außerhalb des Bereiches Verarbeitendes Gewerbe sowie
+Bergbau und Gewinnung von Steinen und Erden liegt.
+Bei 7 kleinbetrieblich strukturierten Branchen gilt eine
+untere Erfassungsgrenze von 10 tätigen Personen.
+Die Auswahl erfolgt jeweils nach dem Beschäftigtenstand Ende
+September des Vorjahres. Die ausgewiesene Beschäftigtenzahl
+betrifft dagegen die von Ende September des Berichtsjahres.
+Die Merkmalswerte beziehen sich auf den gesamten Betrieb,
+schließen damit die nicht produzierenden Betriebsteile mit
+ein.
+Maßgebend für die Zuordnung ist ab 2009 die „Klassifikation
+der Wirtschaftszweige, Ausgabe 2008 (WZ 2008)“, und zwar
+die Abschnitte B und C.
+
+- Datei: `42111-01-04-5.xlsx`
+- Stand: 30.09.2021
+
+## Gebäude mit Wohnraum nach Heizungsart (31211-04-01-5-B)
+
+Zensus 2011: Gebäude mit Wohnraum nach Heizungsart
+
+- Datei: `31211-04-01-5-B.xlsx`
+- Stand: 09.05.2011
+
+## Gebäude mit Wohnraum nach Heizungsart (31231-02-01-5)
+
+Bestand an Wohngebäuden und Wohnungen in Wohn- und Nichtwohngebäuden -
+Fortschreibung auf Basis der endgültigen Ergebnisse der Gebäude- und
+Wohnungszählung 2011 (Zensus 2011).
+
+- Datei: `31231-02-01-5.xlsx`
+- Stand: 31.12.2021
diff --git a/digipipe/store/raw/regiostat/metadata.json b/digipipe/store/raw/regiostat/metadata.json
new file mode 100644
index 00000000..aa6b8315
--- /dev/null
+++ b/digipipe/store/raw/regiostat/metadata.json
@@ -0,0 +1,130 @@
+{
+ "name": "regiostat",
+ "title": "Regionalstatistik (GENESIS)",
+ "id": "regiostat",
+ "description": "Energieverwendung der Betriebe im Verarbeitenden Gewerbe (43531-01-02-4), Betriebe, tätige Personen, Bruttoentgelte (42111-01-04-5), Gebäude mit Wohnraum nach Heizungsart (31211-04-01-5-B), Gebäude mit Wohnraum nach Heizungsart (31231-02-01-5)",
+ "language": [
+ "de-DE"
+ ],
+ "subject": null,
+
+ "keywords": [
+ "Regionalstatistik",
+ "Energieverwendung",
+ "verarbeitendes Gewerbe",
+ "tätige Personen",
+ "Bruttoentgelte",
+ "Gebäude",
+ "Heizungsart"
+ ],
+ "publicationDate": null,
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": "2021-01-01",
+ "timeseries": null
+ },
+ "sources": [
+ {
+ "title": "Regionalstatistik (GENESIS)",
+ "description": "Energieverwendung der Betriebe im Verarbeitenden Gewerbe (43531-01-02-4), Betriebe, tätige Personen, Bruttoentgelte (42111-01-04-5), Gebäude mit Wohnraum nach Heizungsart (31211-04-01-5-B), Gebäude mit Wohnraum nach Heizungsart (31231-02-01-5)",
+ "path": ["https://www.regionalstatistik.de/genesis//online?operation=table&code=43531-01-02-4", "https://www.regionalstatistik.de/genesis//online?operation=table&code=42111-01-04-5"],
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Statistische Ämter des Bundes und der Länder, 2023"
+ }
+ ]
+ },
+ {
+ "title": null,
+ "description": null,
+ "path": null,
+ "licenses": [
+ {
+ "name": null,
+ "title": null,
+ "path": null,
+ "instruction": null,
+ "attribution": null
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": null,
+ "title": null,
+ "path": null,
+ "instruction": null,
+ "attribution": null
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-08-15",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": [],
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/renewables.ninja_feedin/data/.gitkeep b/digipipe/store/raw/renewables.ninja_feedin/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/renewables.ninja_feedin/dataset.md b/digipipe/store/raw/renewables.ninja_feedin/dataset.md
new file mode 100644
index 00000000..bfd272c3
--- /dev/null
+++ b/digipipe/store/raw/renewables.ninja_feedin/dataset.md
@@ -0,0 +1,221 @@
+# EE-Einspeisezeitreihen
+
+Einspeisezeitreihen für Erneuerbare Energien, normiert auf 1 MW bzw. 1 p.u.
+Als Wetterjahr wird 2011 verwendet, siehe
+[Szenarien](../../../../docs/sections/scenarios.md).
+
+## Windenergie
+
+Stündlich aufgelöste Zeitreihe der Windenergie Einspeisung über 1 Jahr auf Basis
+von [MaStR](../bnetza_mastr/dataset.md) und
+[renewables.ninja](http://renewables.ninja).
+Auf einen Auflösung auf Gemeindeebene wird verzichtet, da die Differenz der
+Produktion der Gemeinden nach renewables.ninja <5 % beträgt.
+
+### Windenergieanlage (2022)
+
+Für renewables.ninja sind Position (lat, lon), Nennleistung (capacity),
+Nabenhöhe und Turbinentyp erforderlich.
+
+#### Position
+
+Hierfür wird aus den Zentroiden der Gemeinden ein räumlicher Mittelwert
+anhand des Datensatzes
+[bkg_vg250_muns_region](../../datasets/bkg_vg250_muns_region/dataset.md)
+(`bkg_vg250_muns_region.gpkg`) gebildet:
+
+```
+import geopandas as gpd
+import os.path
+
+def get_position(gdf):
+ df = gpd.read_file(gdf)
+ points_of_muns = df["geometry"].centroid
+ points_of_muns_crs = points_of_muns.to_crs(4326)
+ point_df = [
+ points_of_muns_crs.y.sum()/len(points_of_muns),
+ points_of_muns_crs.x.sum()/len(points_of_muns)
+ ]
+ return point_df
+
+data_folder = os.path.join("your_data_folder")
+muns_gpkg = os.path.join(data_folder, "bkg_vg250_muns_region.gpkg")
+center_position = get_position(muns_gpkg)
+```
+
+#### Nennleistung
+
+Wird auf 1 MW gesetzt/normiert.
+
+#### Nabenhöhe
+
+Aus dem Datensatz
+[bnetza_mastr_wind_region](../../datasets/bnetza_mastr_wind_region/dataset.md)
+(`bnetza_mastr_wind_agg_abw.gpkg`) wird ein Mittelwer von 100 m abgeleitet.
+
+```
+import geopandas as gpd
+
+df = gpd.read_file("bnetza_mastr_wind_agg_abw.gpkg")
+height = df[["hub_height"]].mean()
+```
+
+#### Turbinentyp
+
+Annahme: Innerhalb eines Herstellers sind Leistungskurven sehr ähnlich.
+Daher werden zwei größten Hersteller mit jeweiligen häufigsten Turbinentyp
+ausgewählt - diese sind Enercon und Vestas mit ca. 70 % und ca. 30%.
+
+```
+import geopandas as gpd
+
+df = gpd.read_file("bnetza_mastr_wind_agg_abw.gpkg")
+manufacturers = df[
+ ["manufacturer_name", "status"]
+].groupby("manufacturer_name").count().sort_values(
+ by="status", ascending=False
+)
+```
+
+Häufigste Turbinentypen sind *Enercon E-70* und *Vestas V80*. Daher werden
+*Enercon E70 2000* und *Vestas V80 2000* in renewables.ninja ausgewählt.
+
+```
+man_1 = manufacturers.index[0]
+man_2 = manufacturers.index[1]
+
+type_1 = df[
+ ["manufacturer_name", "type_name", "status"]
+].where(df["manufacturer_name"] == man_1).groupby(
+ "type_name").count().sort_values(by="status", ascending=False)
+
+type_2 = df[
+ ["manufacturer_name", "type_name", "status"]
+].where(df["manufacturer_name"] == man_2).groupby(
+ "type_name").count().sort_values(by="status", ascending=False)
+```
+
+### Raw Data von [renewables.ninja](http://renewables.ninja) API
+
+Es werden zwei Zeitreihen für oben beschriebenen Vergleichsanlagen berechnet:
+
+```
+import json
+import requests
+import pandas as pd
+import geopandas as gpd
+
+def change_wpt(position, capacity, height, turbine):
+ args = {
+ 'lat': 51.8000, # 51.5000-52.0000
+ 'lon': 12.2000, # 11.8000-13.1500
+ 'date_from': '2011-01-01',
+ 'date_to': '2011-12-31',
+ 'capacity': 1000.0,
+ 'height': 100,
+ 'turbine': 'Vestas V164 7000',
+ 'format': 'json',
+ 'local_time': 'true',
+ 'raw': 'false',
+ }
+
+ args['capacity'] = capacity
+ args['height'] = height
+ args['lat'] = position[0]
+ args['lon'] = position[1]
+ args['turbine'] = turbine
+
+ return args
+
+def get_df(args):
+ token = 'Please get your own'
+ api_base = 'https://www.renewables.ninja/api/'
+
+ s = requests.session()
+ # Send token header with each request
+ s.headers = {'Authorization': 'Token ' + token}
+
+ url = api_base + 'data/wind'
+
+ r = s.get(url, params=args)
+
+ parsed_response = json.loads(r.text)
+ df = pd.read_json(
+ json.dumps(parsed_response['data']),orient='index')
+ metadata = parsed_response['metadata']
+ return df
+
+enercon_production = get_df(change_wpt(
+ position,
+ capacity=1,
+ height=df[["hub_height"]].mean(),
+ turbine=enercon)
+)
+
+vestas_production = get_df(change_wpt(
+ position,
+ capacity=1000,
+ height=df[["hub_height"]].mean(),
+ turbine=vestas)
+)
+```
+
+### Gewichtung und Skalierung der Zeitreihen
+
+Um die Charakteristika der beiden o.g. Anlagentypen zu berücksichtigen, erfolgt
+eine gewichtete Summierung der Zeitreihen anhand der berechneten Häufigkeit.
+
+### Zukunftsszenarien
+
+Analog zu dem oben beschriebenen Vorgehen wird eine separate Zeitreihe für
+zukünftige WEA berechnet. Hierbei wird eine Enercon E126 6500 mit einer
+Nabenhöhe von 159 m angenommen
+([PV- und Windflächenrechner](https://zenodo.org/record/6794558)).
+
+Da die Zeitreihe sich nur marginal von der obigen Status-quo-Zeitreihe
+unterscheidet, wird letztere sowohl für den Status quo als auch die
+Zukunftsszenarien verwendet.
+
+- Einspeisezeitreihe: `wind_feedin_timeseries.csv`
+
+## Freiflächen-Photovoltaik
+
+### PV-Anlage (2022)
+
+Stündlich aufgelöste Zeitreihe der Photovoltaikeinspeisung über 1 Jahr auf Basis
+von [MaStR](../bnetza_mastr/dataset.md) und
+[renewables.ninja](http://renewables.ninja).
+Wie bei der Windeinspeisung wird auf eine Auflsöung auf Gemeindeebene aufgrund
+geringer regionaler Abweichungen verzichtet.
+
+Für die Generierung der Zeitreihe über
+[renewables.ninja](http://renewables.ninja)
+wird eine Position(lat, lon), Nennleistung (capacity), Verluste (system_loss)
+Nachführung (tracking), Neigung (tilt) und der Azimutwinkel (azim) benötigt.
+
+Als Position wird analog zur Windenergieanlage der räumlicher Mittelwert
+verwendet. Laut MaStR werden lediglich 13 Anlagen nachgeführt (0,01 % der
+Kapazität), die Nachführung wird daher vernachlässigt. Die Neigung ist aus MaStR
+nicht bekannt, es dominieren jedoch Anlagen auf Freiflächen sowie Flachdächern
+im landwirtschaftlichen Kontext. Nach
+[Ariadne Szenarienreport](https://ariadneprojekt.de/media/2022/02/Ariadne_Szenarienreport_Oktober2021_corr0222_lowres.pdf)
+wird diese mit 30° angenommen.
+Die Nennleistung Wird auf 1 MW gesetzt/normiert.
+
+### Zukunftsszenarien
+
+Die Status-quo-Zeitreihe wird sowohl für den Status quo als auch die
+Zukunftsszenarien verwendet.
+
+- Einspeisezeitreihe: `pv_feedin_timeseries.csv`
+
+## Solarthermie
+
+- Einspeisezeitreihe: `st_feedin_timeseries.csv` (Kopie von
+ PV-Einspeisezeitreihe)
+
+## Laufwasserkraft
+
+Hier wird eine konstante Einspeisung angenommen.
+
+- Einspeisezeitreihe: `ror_feedin_timeseries.csv`
diff --git a/digipipe/store/raw/renewables.ninja_feedin/metadata.json b/digipipe/store/raw/renewables.ninja_feedin/metadata.json
new file mode 100644
index 00000000..fed94b1b
--- /dev/null
+++ b/digipipe/store/raw/renewables.ninja_feedin/metadata.json
@@ -0,0 +1,119 @@
+{
+ "name": "renewables.ninja_feedin",
+ "title": "EE-Einspeisezeitreihen",
+ "id": "renewables.ninja_feedin",
+ "description": "Einspeisezeitreihen für Erneuerbare Energien, normiert auf 1 MW bzw. 1 p.u. Als Wetterjahr wird 2011 verwendet",
+ "language": [
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "Erneuerbare",
+ "Energien",
+ "Einspeisezeitreihen",
+ "renewables.ninja"
+ ],
+ "publicationDate": "2016-09-21",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": null,
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": "2023-04-14",
+ "timeseries": [
+ {
+ "start": "2011-01-01T00:00+01",
+ "end": "2011-12-31T23:00+01",
+ "resolution": "1 h",
+ "alignment": "left"
+ }
+ ]
+ },
+ "sources": [
+ {
+ "title": "renewables.ninja_feedin",
+ "description": "Einspeisezeitreihen für Erneuerbare Energien, normiert auf 1 MW bzw. 1 p.u. Als Wetterjahr wird 2011 verwendet",
+ "path": "hhttps://www.renewables.ninja/",
+ "licenses": [
+ {
+ "name": "CC BY-NC 4.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://creativecommons.org/licenses/by-nc/4.0/",
+ "instruction": "you are free to copy, redistribute and adapt them for non-commercial purposes, provided you give appropriate credit. Note that the data is made available as-is and without warranty. We cannot guarantee its accuracy, and accept no responsibility for any liability arising from its use. You are advised to examine the quality of the data for your intended purposes, and to consult the publications linked on this page.",
+ "attribution": "© www.renewables.ninja, 2023"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": "CC BY-NC 4.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://creativecommons.org/licenses/by-nc/4.0/",
+ "instruction": "you are free to copy, redistribute and adapt them for non-commercial purposes, provided you give appropriate credit. Note that the data is made available as-is and without warranty. We cannot guarantee its accuracy, and accept no responsibility for any liability arising from its use. You are advised to examine the quality of the data for your intended purposes, and to consult the publications linked on this page.",
+ "attribution": "© www.renewables.ninja, 2023"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-09-07",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": null,
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+ }
diff --git a/digipipe/store/raw/rli_pv_wfr/data/.gitkeep b/digipipe/store/raw/rli_pv_wfr/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/rli_pv_wfr/dataset.md b/digipipe/store/raw/rli_pv_wfr/dataset.md
new file mode 100644
index 00000000..36a125e6
--- /dev/null
+++ b/digipipe/store/raw/rli_pv_wfr/dataset.md
@@ -0,0 +1,14 @@
+# Geodaten PV- und Windflächenrechner
+
+Geodaten aus dem [PV- und Windflächenrechner](https://www.agora-energiewende.de/service/pv-und-windflaechenrechner/).
+
+Mehr Informationen:
+
+- [Begleitdokument](https://zenodo.org/record/6794558)
+- [Geodaten Potenzialflächen](https://zenodo.org/record/6728382)
+
+Enthält:
+
+- Geodaten
+- Metadaten
+- App-Datapackage
diff --git a/digipipe/store/raw/rli_pv_wfr/metadata.json b/digipipe/store/raw/rli_pv_wfr/metadata.json
new file mode 100644
index 00000000..410f25c2
--- /dev/null
+++ b/digipipe/store/raw/rli_pv_wfr/metadata.json
@@ -0,0 +1,119 @@
+{
+ "name": "rli_pv_wfr",
+ "title": "Geodaten PV- und Windflächenrechner",
+ "id": "rli_pv_wfr",
+ "description": "Geodaten aus dem PV- und Windflächenrechner",
+ "language": [
+ "de-DE"
+ ],
+ "subject": null,
+
+ "keywords": [
+ "Geodaten",
+ "PV-Flächenrechner",
+ "Windflächenrechner",
+ "Potentialflächen"
+ ],
+ "publicationDate": "2022-06-05",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Germany",
+ "extent": "Germany",
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": "2022-06-05",
+ "timeseries": null
+ },
+ "sources": [
+ {
+ "title": "Geodaten PV- und Windflächenrechner",
+ "description": "Geodaten aus dem PV- und Windflächenrechner",
+ "path": "https://zenodo.org/record/6728382",
+ "licenses": null
+ },
+ {
+ "title": null,
+ "description": null,
+ "path": null,
+ "licenses":[
+ {
+ "name": "CC BY-NC 4.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://creativecommons.org/licenses/by-nc/4.0/",
+ "instruction": "you are free to copy, redistribute and adapt them for non-commercial purposes, provided you give appropriate credit. Note that the data is made available as-is and without warranty. We cannot guarantee its accuracy, and accept no responsibility for any liability arising from its use. You are advised to examine the quality of the data for your intended purposes, and to consult the publications linked on this page.",
+ "attribution": "© Reiner Lemoine INstitut, 2022"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": "CC BY-NC 4.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://creativecommons.org/licenses/by-nc/4.0/",
+ "instruction": "you are free to copy, redistribute and adapt them for non-commercial purposes, provided you give appropriate credit. Note that the data is made available as-is and without warranty. We cannot guarantee its accuracy, and accept no responsibility for any liability arising from its use. You are advised to examine the quality of the data for your intended purposes, and to consult the publications linked on this page.",
+ "attribution": "© Reiner Lemoine INstitut, 2022"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-08-15",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": [],
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/rpg_abw_pv_roof_potential/data/.gitkeep b/digipipe/store/raw/rpg_abw_pv_roof_potential/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/rpg_abw_pv_roof_potential/dataset.md b/digipipe/store/raw/rpg_abw_pv_roof_potential/dataset.md
new file mode 100644
index 00000000..4ed0119a
--- /dev/null
+++ b/digipipe/store/raw/rpg_abw_pv_roof_potential/dataset.md
@@ -0,0 +1,19 @@
+# Dachflächenpotenzial PV-Aufdachanlagen in ABW
+
+Abschätzung der installierten Leistung und des Ertrags von PV-Aufdachanlagen in
+Anhalt-Bitterfeld-Wittenberg der Regionalen Planungsgemeinschaft.
+
+Dafür wurden auf Basis des
+[Digitalen Oberflächenmodells (DOM2)](https://www.lvermgeo.sachsen-anhalt.de/de/dom2-landesweit.html)
+Schattenberechnungen durchgeführt. Anhand des
+[LoD2 3D-Gebäudemodells](https://www.lvermgeo.sachsen-anhalt.de/de/download_lod2.html)
+wurden für verschiedene Dachausrichtungen (nord, ost, süd, west, flach) die
+installierbare Leistung bestimmt und mittels der Globalstrahlung und typischer
+technischer Parameter für jedes Gebäude und jede Dachflächenorientierung
+potenzielle Erträge berechnet.
+
+Quellen:
+
+- [Hauptseite](https://www.planungsregion-abw.de/geodaten/)
+- [Geodaten](https://gis-entwicklung2.planungsregion-abw.de/geoserver/wfs?SERVICE=WFS&REQUEST=GetCapabilities)
+- [Anwendung](https://ris.planungsregion-abw.de/mapbender/application/pv_dachflaechenpot_rpg_abw)
diff --git a/digipipe/store/raw/rpg_abw_pv_roof_potential/metadata.json b/digipipe/store/raw/rpg_abw_pv_roof_potential/metadata.json
new file mode 100644
index 00000000..621e632c
--- /dev/null
+++ b/digipipe/store/raw/rpg_abw_pv_roof_potential/metadata.json
@@ -0,0 +1,112 @@
+{
+ "name": "rpg_abw_pv_roof_potential",
+ "title": "Dachflächenpotenzial PV-Aufdachanlagen in ABW",
+ "id": "rpg_abw_pv_roof_potential",
+ "description": "Abschätzung der installierten Leistung und des Ertrags von PV-Aufdachanlagen in Anhalt-Bitterfeld-Wittenberg der Regionalen Planungsgemeinschaft.",
+ "language": [
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "PV",
+ "Photovoltaic",
+ "Flächenpotential",
+ "Aufdachanlagen"
+ ],
+ "publicationDate": null,
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": null,
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "ABW",
+ "extent": "ABW",
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": "2022-06-10",
+ "timeseries": null
+ },
+ "sources": [
+ {
+ "title": "Dachflächenpotenzial PV-Aufdachanlagen in ABW",
+ "description": "Abschätzung der installierten Leistung und des Ertrags von PV-Aufdachanlagen in Anhalt-Bitterfeld-Wittenberg der Regionalen Planungsgemeinschaft.",
+ "path": "https://www.planungsregion-abw.de/geodaten/",
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Regionale Planungsgemeinschaft Anhalt-Bitterfeld-Wittenberg / 2022"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Regionale Planungsgemeinschaft Anhalt-Bitterfeld-Wittenberg / 2022"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-09-07",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": null,
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/rpg_abw_regional_plan/data/.gitkeep b/digipipe/store/raw/rpg_abw_regional_plan/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/rpg_abw_regional_plan/dataset.md b/digipipe/store/raw/rpg_abw_regional_plan/dataset.md
new file mode 100644
index 00000000..663538b7
--- /dev/null
+++ b/digipipe/store/raw/rpg_abw_regional_plan/dataset.md
@@ -0,0 +1,42 @@
+# Regionalplan Anhalt-Bitterfeld-Wittenberg
+
+Geodatensätze aus Teilplänen Wind 2018 und 2027 der Regionalen
+Planungsgemeinschaft Anhalt-Bitterfeld-Wittenberg.
+
+## Sachlicher Teilplan Wind 2018
+
+Geodaten aus rechtskräftigem
+[Sachlichen Teilplan Wind 2018](https://www.planungsregion-abw.de/regionalplanung/teilplan-windenergie/teilplan-2018/).
+
+> Im Sachlichen Teilplan "Nutzung der Windenergie in der Planungsregion
+> Anhalt-Bitterfeld-Wittenberg" vom 30.05.2018 werden 22 Vorranggebiete für die
+> Nutzung der Windenergie mit der Wirkung von Eignungsgebieten festgelegt. Sie
+> dienen der raumordnerischen Steuerung der Errichtung von raumbedeutsamen
+> Windenergieanlagen in Konzentrationszonen.
+>
+> Die oberste Landesentwicklungsbehörde hat am 01.08.2018 die Genehmigung
+> erteilt. Mit Bekanntmachung der Genehmigung tritt der Sachliche Teilplan in
+> Kraft.
+
+Dateien:
+
+- Vorrang-/Eignungsgebiete: `stp_2018_vreg.gpkg`
+ ([Quelle](https://gis.planungsregion-abw.de/geoserver/stp_wind2018/ows?SERVICE=WFS&REQUEST=GetCapabilities))
+
+## Sachlicher Teilplan Wind 2027
+
+Geodaten aus Planentwurf des
+[Sachlichen Teilplan Wind 2027](https://www.planungsregion-abw.de/regionalplanung/teilplan-windenergie/teilplan-2027/).
+
+> Die Regionalversammlung hat am 03.03.2023 beschlossen, den Sachlichen
+> Teilplan "Windenergie 2027 in der Planungsregion Anhalt-Bitterfeld-Wittenberg"
+> aufzustellen und mit der Bekanntgabe der Allgemeinen Planungsabsicht die
+> beabsichtigten Auswahlkriterien und mögliche Gebietskulisse der Vorranggebiete
+> für die Nutzung der Windenergie bzw. für Repowering von Windenergieanlagen
+> vorzustellen.
+
+Dateien:
+
+- Suchräume: `stp_2027_suchraum.gpkg` (Quelle: RPG ABW)
+- Planabsicht Vorranggebiete: `stp_2027_ideen_vr.gpkg` (Quelle: RPG ABW)
+- Planabsicht Repoweringgebiete: `stp_2027_ideen_repower.gpkg` (Quelle: RPG ABW)
diff --git a/digipipe/store/raw/rpg_abw_regional_plan/metadata.json b/digipipe/store/raw/rpg_abw_regional_plan/metadata.json
new file mode 100644
index 00000000..585f2deb
--- /dev/null
+++ b/digipipe/store/raw/rpg_abw_regional_plan/metadata.json
@@ -0,0 +1,111 @@
+{
+ "name": "rpg_abw_regional_plan",
+ "title": "Regionalplan Anhalt-Bitterfeld-Wittenberg",
+ "id": "rpg_abw_regional_plan",
+ "description": "Geodatensätze aus Teilplänen Wind 2018 und 2027 der Regionalen Planungsgemeinschaft Anhalt-Bitterfeld-Wittenberg.",
+ "language": [
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "Geodatensätze",
+ "Teilpläne",
+ "PLanungsgemeinschaft"
+ ],
+ "publicationDate": "2018-05-30",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": null,
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "ABW",
+ "extent": "ABW",
+ "resolution": null
+ },
+ "temporal": {
+ "referenceDate": "2023-03-03",
+ "timeseries": null
+ },
+ "sources": [
+ {
+ "title": "Regionalplan Anhalt-Bitterfeld-Wittenberg",
+ "description": "Geodatensätze aus Teilplänen Wind 2018 und 2027 der Regionalen Planungsgemeinschaft Anhalt-Bitterfeld-Wittenberg.",
+ "path": ["https://www.planungsregion-abw.de/regionalplanung/teilplan-windenergie/teilplan-2018/", "https://www.planungsregion-abw.de/regionalplanung/teilplan-windenergie/teilplan-2027/"],
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Regionale Planungsgemeinschaft Anhalt-Bitterfeld-Wittenberg / Jahr"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Regionale Planungsgemeinschaft Anhalt-Bitterfeld-Wittenberg / Jahr"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-09-07",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": null,
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/seenergies_peta5/data/.gitkeep b/digipipe/store/raw/seenergies_peta5/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/seenergies_peta5/dataset.md b/digipipe/store/raw/seenergies_peta5/dataset.md
new file mode 100644
index 00000000..24118e8e
--- /dev/null
+++ b/digipipe/store/raw/seenergies_peta5/dataset.md
@@ -0,0 +1,26 @@
+# sEEnergies Pan-European Thermal Atlas 5.2 (Peta5)
+
+Wärmebedarf für Europa 2015 in GJ (1ha Auflösung) für
+
+- Haushalte: Raumwärme und Warmwasser
+- GHD: Raumwärme, Warmwasser und Prozesswärme
+
+Die Daten können auf der
+[Projektseite](https://s-eenergies-open-data-euf.hub.arcgis.com)
+eingesehen werden.
+
+## Haushalte
+
+Abgerufen mittels
+
+```commandline
+wget -O Peta5_0_1_HD_res.zip https://arcgis.com/sharing/rest/content/items/d7d18b63250240a49eb81db972aa573e/data
+```
+
+## GHD und Industrie
+
+Abgerufen mittels
+
+```commandline
+wget -O Peta5_0_1_HD_ser.zip https://arcgis.com/sharing/rest/content/items/52ff5e02111142459ed5c2fe3d80b3a0/data
+```
diff --git a/digipipe/store/raw/seenergies_peta5/metadata.json b/digipipe/store/raw/seenergies_peta5/metadata.json
new file mode 100644
index 00000000..508362a4
--- /dev/null
+++ b/digipipe/store/raw/seenergies_peta5/metadata.json
@@ -0,0 +1,112 @@
+{
+ "name": "seenergies_peta5",
+ "title": "sEEnergies Pan-European Thermal Atlas 5.2 (Peta5)",
+ "id": "seenergies_peta5",
+ "description": "Wärmebedarf für Europa 2015 in GJ (1ha Auflösung)",
+ "language": [
+ "en-GB"
+ ],
+ "subject": null,
+ "keywords": [
+ "European",
+ "Photovoltaic",
+ "Flächenpotential",
+ "Aufdachanlagen"
+ ],
+ "publicationDate": "2022-01-01",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": null,
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Europe",
+ "extent": "Europe",
+ "resolution": "1 ha"
+ },
+ "temporal": {
+ "referenceDate": "2022-01-01",
+ "timeseries": null
+ },
+ "sources": [
+ {
+ "title": "sEEnergies Pan-European Thermal Atlas 5.2 (Peta5)",
+ "description": "Wärmebedarf für Europa 2015 in GJ (1ha Auflösung)",
+ "path": "https://www.seenergies.eu/peta5/",
+ "licenses": [
+ {
+ "name": null,
+ "title": null,
+ "path": null,
+ "instruction": "The data provided is indiative and for research purpose only",
+ "attribution": "© Flensburg, Halmstad and Aalborg Universities 2022"
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": null,
+ "title": null,
+ "path": null,
+ "instruction": "The data provided is indiative and for research purpose only",
+ "attribution": "© Flensburg, Halmstad and Aalborg Universities 2022"
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-09-07",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": null,
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/stala_st_energy/data/.gitkeep b/digipipe/store/raw/stala_st_energy/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/stala_st_energy/dataset.md b/digipipe/store/raw/stala_st_energy/dataset.md
new file mode 100644
index 00000000..3f7a236e
--- /dev/null
+++ b/digipipe/store/raw/stala_st_energy/dataset.md
@@ -0,0 +1,10 @@
+# Energiedaten Sachsen-Anhalt
+
+Datensätze zur Energie- und Wasserversorgung des Statistischen Landesamtes
+Sachsen-Anhalt.
+
+## Daten
+
+Stromverbrauch der Industriebetriebe nach Kreisen 2003-2021 in MWh
+
+- [Quelle](https://statistik.sachsen-anhalt.de/themen/wirtschaftsbereiche/energie-und-wasserversorgung/tabellen-energieverwendung#c206986)
diff --git a/digipipe/store/raw/stala_st_energy/metadata.json b/digipipe/store/raw/stala_st_energy/metadata.json
new file mode 100644
index 00000000..a912d129
--- /dev/null
+++ b/digipipe/store/raw/stala_st_energy/metadata.json
@@ -0,0 +1,118 @@
+{
+ "name": "stala_st_energy",
+ "title": "Energiedaten Sachsen-Anhalt",
+ "id": "stala_st_energy",
+ "description": "Datensätze zur Energie- und Wasserversorgung des Statistischen Landesamtes Sachsen-Anhalt.",
+ "language": [
+ "de-DE"
+ ],
+ "subject": null,
+ "keywords": [
+ "Energiedaten",
+ "Energieversorgung",
+ "Wasserversorgung"
+ ],
+ "publicationDate": "2022-01-01",
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": null,
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Sachsen-Anhalt",
+ "extent": "Sachsen-Anhalt",
+ "resolution": "NUTS-3"
+ },
+ "temporal": {
+ "referenceDate": "2022-01-01",
+ "timeseries": [
+ {
+ "start": "2003-01-01T00:00+01",
+ "end": "2021-12-31T23:00+01",
+ "resolution": "1 a",
+ "alignment": "left"
+ }
+ ]
+ },
+ "sources": [
+ {
+ "title": "Energiedaten Sachsen-Anhalt",
+ "description": "Datensätze zur Energie- und Wasserversorgung des Statistischen Landesamtes Sachsen-Anhalt.",
+ "path": "https://statistik.sachsen-anhalt.de/themen/wirtschaftsbereiche/energie-und-wasserversorgung/tabellen-energieverwendung#c206986",
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Statistisches Landesamt Sachsen-Anhalt, Halle (Saale)."
+ }
+ ]
+ }
+ ],
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© Statistisches Landesamt Sachsen-Anhalt, Halle (Saale)."
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-09-07",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": null,
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": null,
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+ },
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/stala_st_pop_prog/data/.gitkeep b/digipipe/store/raw/stala_st_pop_prog/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/stala_st_pop_prog/dataset.md b/digipipe/store/raw/stala_st_pop_prog/dataset.md
new file mode 100644
index 00000000..78cb55a7
--- /dev/null
+++ b/digipipe/store/raw/stala_st_pop_prog/dataset.md
@@ -0,0 +1,4 @@
+# Bevölkerungsprognose Sachsen-Anhalt
+
+Bevölkerungsprognose je Gemeinde bis 2035 des Statistischen Landesamtes
+Sachsen-Anhalt. Stand: 2021
diff --git a/digipipe/store/raw/stala_st_pop_prog/metadata.json b/digipipe/store/raw/stala_st_pop_prog/metadata.json
new file mode 100644
index 00000000..017ab676
--- /dev/null
+++ b/digipipe/store/raw/stala_st_pop_prog/metadata.json
@@ -0,0 +1,109 @@
+{
+ "name": "stala_st_pop_prog",
+ "title": "Regionalisierte Bevölkerungsprognose",
+ "id": "stala_st_pop_prog",
+ "description": "Prognostizierter Bevölkerungsstand in den Gemeinden, kreisfreien Städten und Landkreisen nach Prognosejahr und Geschlecht",
+ "language": [
+ "de-DE"
+ ],
+ "subject": [],
+ "keywords": [
+ "Bevölkerungsprognose",
+ "population"
+ ],
+ "publicationDate": null,
+ "context": {
+ "homepage": "https://abw.rl-institut.de",
+ "documentation": "https://digiplan.readthedocs.io",
+ "sourceCode": "https://github.com/rl-institut/digipipe/",
+ "contact": "https://reiner-lemoine-institut.de/ueber-uns/kontakt/",
+ "grantNo": "None",
+ "fundingAgency": "https://www.region-gestalten.bund.de",
+ "fundingAgencyLogo": "https://www.region-gestalten.bund.de/Region/SiteGlobals/Frontend/Images/logo.svg",
+ "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png"
+ },
+ "spatial": {
+ "location": "Sachsen-Anhalt",
+ "extent": "Sachsen-Anhalt",
+ "resolution": ""
+ },
+ "temporal": {
+ "referenceDate": null,
+ "timeseries": [
+ {
+ "start": "2019",
+ "end": "2035",
+ "resolution": "1 year",
+ "alignment": null,
+ "aggregationType": "sum"
+ }
+ ]
+ },
+ "sources": [
+ {
+ "title": "1_Internettabelle_7RBP_nach_Prognosejahr_Geschlecht_alle_Ebenen",
+ "description": "Prognostizierter Bevölkerungsstand in den Gemeinden, kreisfreien Städten und Landkreisen nach Prognosejahr und Geschlecht",
+ "path": "statistik.sachsen-anhalt.de/themen/bevoelkerung-mikrozensus-freiwillige-haushaltserhebungen/bevoelkerung/bevoelkerungsprognose-und-haushalteprognose/#c312231",
+ "licenses": [
+ {
+ "name": "DL-DE-BY-2.0",
+ "title": "Data licence Germany – attribution – Version 2.0",
+ "path": "https://www.govdata.de/dl-de/by-2-0",
+ "instruction": "The data and meta-data provided may, for commercial and non-commercial use, in particular be copied, printed, presented, altered, processed and transmitted to third parties; be merged with own data and with the data of others and be combined to form new and independent datasets; be integrated in internal and external business processes, products and applications in public and non-public electronic networks.",
+ "attribution": "© 2023 Landesportal Sachsen-Anhalt "
+ }
+ ]
+ }
+ ],
+ "contributors": [
+ {
+ "title": "aaronschilling",
+ "email": "Aaron.Schilling@rl-institut.de",
+ "date": "2023-09-07",
+ "object": "metadata",
+ "comment": "create metadata"
+ }
+ ],
+ "resources": [
+ {
+ "profile": null,
+ "name": null,
+ "path": null,
+ "format": null,
+ "encoding": null,
+ "schema": {
+ "fields": [],
+ "primaryKey": [],
+ "foreignKeys": []
+ },
+ "dialect": {
+ "delimiter": "",
+ "decimalSeparator": "."
+ }
+ }
+ ],
+ "@id": [],
+ "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+ "review": {
+ "path": "",
+ "badge": ""
+ },
+ "metaMetadata": {
+ "metadataVersion": "OEP-1.5.2",
+ "metadataLicense": {
+ "name": "CC0-1.0",
+ "title": "Creative Commons Zero v1.0 Universal",
+ "path": "https://creativecommons.org/publicdomain/zero/1.0/"
+ }
+},
+ "_comment": {
+ "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+ "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
+ "units": "Use a space between numbers and units (100 m)",
+ "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+ "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+ "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+ "null": "If not applicable use: null",
+ "todo": "If a value is not yet available, use: todo"
+ }
+}
diff --git a/digipipe/store/raw/technology_data/data/.gitkeep b/digipipe/store/raw/technology_data/data/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/digipipe/store/raw/technology_data/dataset.md b/digipipe/store/raw/technology_data/dataset.md
new file mode 100644
index 00000000..fc663b27
--- /dev/null
+++ b/digipipe/store/raw/technology_data/dataset.md
@@ -0,0 +1,1410 @@
+# Technologiedaten
+
+## Jahresvolllaststunden
+
+Anhand typischer heutiger und prognostizierter Werte für Sachsen-Anhalt werden
+folgende Jahresvolllaststunden angenommen:
+
+| Technologie | Jahr | Volllaststunden | Quelle(n) für Annahme | Anmerkung |
+|-----------------|------|----------------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------|
+| Windenergie | 2022 | 1800 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/wind/auswahl/811-durchschnittliche_ja/#goto_811) | |
+| | 2045 | 2300 | [PV- und Windflächenrechner](https://zenodo.org/record/6794558) | |
+| Freiflächen-PV | 2022 | 980 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/solar/auswahl/813-durchschnittliche_ja/#goto_813), [ISE](https://www.ise.fraunhofer.de/content/dam/ise/de/documents/publications/studies/aktuelle-fakten-zur-photovoltaik-in-deutschland.pdf) | |
+| | 2045 | 980 | [PV- und Windflächenrechner](https://zenodo.org/record/6794558), [Ariadne Szenarienreport](https://ariadneprojekt.de/media/2022/02/Ariadne_Szenarienreport_Oktober2021_corr0222_lowres.pdf) | |
+| Aufdach-PV | 2022 | 910 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/solar/auswahl/813-durchschnittliche_ja/#goto_813), [ISE](https://www.ise.fraunhofer.de/content/dam/ise/de/documents/publications/studies/aktuelle-fakten-zur-photovoltaik-in-deutschland.pdf) | |
+| | 2045 | 910 | [Ariadne Szenarienreport](https://ariadneprojekt.de/media/2022/02/Ariadne_Szenarienreport_Oktober2021_corr0222_lowres.pdf) | |
+| Laufwasserkraft | 2022 | 3800 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/wasser/auswahl/840-durchschnittliche_ja/#goto_840) | |
+| | 2045 | 3800 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/wasser/auswahl/840-durchschnittliche_ja/#goto_840) | |
+| Bioenergie | 2022 | 6000 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/bioenergie/auswahl/814-durchschnittliche_ja/#goto_814), [ISE](https://www.ise.fraunhofer.de/content/dam/ise/de/documents/publications/studies/DE2018_ISE_Studie_Stromgestehungskosten_Erneuerbare_Energien.pdf) | Bioenergie-Stromerzeugung (ohne update_raw
+
+A zip file from a prespecified
+[URL](https://wolke.rl-institut.de/s/aN2ccptGtFsFiDs/download)
+is downloaded and unzipped to `store/temp/`. The raw data files are copied to
+the corresponding folders in `store/raw/`.
+A prompt asks if an already existing file should be updated. Confirm with "y"
+or type "n" to skip.
+
+The following additional files must be downloaded manually:
+
+- [OpenStreetMap](https://download.geofabrik.de/europe/germany-230101.osm.pbf)
+ --> place in `store/raw/osm/data/`
+
+## Run
+
+To run the pipeline, go to Digipipe's root `digipipe/` or to
+`digipipe/workflow/` and type
+
+ snakemake -j
+
+while `NUMBER_OF_CPU_CORES` is the number of CPU cores to be used for the
+pipeline execution. You can also make a dry-run (see what snakemake would do
+but without actually really doing anything) by typing
+
+ snakemake -n
+
+To clean all produced data, use
+
+ snakemake -j1 clean
+
+This involves preprocessed data in directories: preprocessed, datasets and
+appdata.
+
+## Pipeline visualization / DAG
+
+The entire pipeline can be visualized as a directed acyclic graph (DAG).
+The following command creates the DAG as an svg file in the current directory:
+
+ snakemake --dag | dot -Tsvg > dag_rules_full.svg
+
+As the full graph is too packed with information and therefore hardly to grasp,
+consider to show only certain parts by disabling some target files in the `all`
+rule. Also, a simple rule graph (the one shown above) can be created and saved
+in the current directory using
+
+ snakemake --rulegraph | dot -Tsvg > dag_rules_simple.svg
+
+To create a graph in the current directory showing the file dependencies, type
+
+ snakemake --filegraph | dot -Tsvg > dag_files.svg
+
+The graphs also provide information on the completed (solid lines) and pending
+(dashed lines) processing steps. For further details see
+[Snakemake CLI docs](https://snakemake.readthedocs.io/en/stable/executing/cli.html).
+
+## Snakefiles and config
+
+- The global workflow is defined in the main
+ [Snakefile](../workflow/Snakefile).
+- It includes the module Snakefiles from the data store located at
+ - [store/preprocessed/module.smk](../store/preprocessed/module.smk) and
+ - [store/datasets/module.smk](../store/datasets/module.smk)
+- In each of these modules, the rules as well as the config from the contained
+ datasets are imported.
diff --git a/digipipe/workflow/utils.py b/digipipe/workflow/utils.py
new file mode 100644
index 00000000..b4702385
--- /dev/null
+++ b/digipipe/workflow/utils.py
@@ -0,0 +1,23 @@
+"""
+Helper functions for the workflow
+"""
+
+
+def search_data_workflows():
+ """Search for snakefiles (*.smk) in data store but exclude templates.
+
+ TODO: Used in outdated manual importing of snakemake files.
+ TODO: Remove, if modularization works out.
+
+ Returns
+ -------
+ list
+ Paths to Snakemake files
+ """
+ # smk_files = []
+ # for root, dirs, files in os.walk(get_abs_store_root_path()):
+ # for file in files:
+ # if file.endswith(".smk") and ".TEMPLATE" not in str(root):
+ # smk_files.append(Path(os.path.join(root, file)))
+ # return smk_files
+ raise NotImplementedError
diff --git a/docs/datasets/appdata_datasets.md b/docs/datasets/appdata_datasets.md
new file mode 100644
index 00000000..e8014915
--- /dev/null
+++ b/docs/datasets/appdata_datasets.md
@@ -0,0 +1,31 @@
+# 'Appdata' Datasets
+
+------------------------------
+## Datapackage für App
+
+Von der App benötigte Daten in JSON `datapackage_app.json`.
+
+Generelle Struktur:
+
+- ``
+ - `` (`scalars`, `sequences` oder `geodata`)
+ - ``
+ - `description`: Beschreibung
+ - `path`: Pfad zur Zieldatei im Datapackage
+ - `fields`: Felder-/Spaltendefinition
+ - `_source_path`: Pfad zur Datei im Quelldatensatz
+ - `dataset`: Name in `store/datasets/`
+ - `file`: Datei
+
+Kategorien bzw. Inhalt `resources`:
+
+- `base_data`: Basisdaten
+- `production`: Energiebereitstellung
+- `demand`: Energiebedarf
+- `potential_areas`: Potenzialgebiete EE
+- `emissions`: Emissionen
+- `settings`: App-Settings
+- `captions`: App-Captions
+
+**Dataset: `appdata/datapackage`**
+
diff --git a/docs/datasets/datasets_datasets.md b/docs/datasets/datasets_datasets.md
new file mode 100644
index 00000000..ff920b83
--- /dev/null
+++ b/docs/datasets/datasets_datasets.md
@@ -0,0 +1,945 @@
+# 'Datasets' Datasets
+
+------------------------------
+## Technologiedaten
+
+Allgemeine Technologiedaten.
+
+Raw dataset: [technology_data](../../digipipe/store/raw/technology_data/dataset.md)
+
+**Dataset: `datasets/technology_data`**
+
+
+------------------------------
+## Potenzialgebiete Windenergie
+
+Potenzialgebiete für die Errichtung von Windenergieanlagen, basierend auf den
+Teilplänen Wind der Regionalen Planungsgemeinschaft Anhalt-Bitterfeld-Wittenberg
+aus
+[rpg_abw_regional_plan](../../digipipe/store/preprocessed/rpg_abw_regional_plan/dataset.md).
+
+Dateien:
+
+- STP Wind 2018 - Vorrang-/Eignungsgebiete:
+ `potentialarea_wind_stp_2018_vreg.gpkg`
+- STP Wind 2027 - Planabsicht Vorranggebiete:
+ `potentialarea_wind_stp_2027_vr.gpkg`
+- STP Wind 2027 - Planabsicht Repoweringgebiete:
+ `potentialarea_wind_stp_2027_repowering.gpkg`
+- STP Wind 2027 - Suchraum Wald:
+ `potentialarea_wind_stp_2027_search_area_forest_area.gpkg`
+- STP Wind 2027 - Suchraum Offenland:
+ `potentialarea_wind_stp_2027_search_area_open_area.gpkg`
+
+Die darin verwendeten Attributtexte werden in die Datei
+`potentialarea_wind_attribute_captions.json` exportiert.
+
+Die Flächen werden mit den Gemeindegrenzen verschnitten und den Gemeinden
+zugeordnet. Je Gemeinde und obigem Flächentyp/Datei wird eine Flächensumme (in
+km²) berechnet, siehe `potentialarea_wind_area_stats_muns.csv`. Die Gemeinden
+werden über den Schlüssel `municipality_id` (vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_muns_region/dataset.md))
+identifiziert.
+
+**Dataset: `datasets/potentialarea_wind_region`**
+
+
+------------------------------
+## Speicheranlagen
+
+Speicheranlagen in der Region aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Es wird weiterhin geprüft, ob dem Speicher eine oder mehrere PV-Aufdachanlagen
+zugeordnet sind, es wird die Anzahl und Summe der Nettonennleistung berechnet.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Weiterhin erfolgt eine Auswertung der installierten Gesamtleistung je Gemeinde:
+
+- Alle Speicher: `bnetza_mastr_storage_stats_muns.csv`
+- Großspeicher (>=100 kWh): `bnetza_mastr_storage_large_stats_muns.csv`
+- Kleinspeicher (<100 kWh): `bnetza_mastr_storage_small_stats_muns.csv`
+
+`bnetza_mastr_storage_pv_roof.json` enthält die spezifische Speicherkapazität
+sowie spezifische Nennleistung der Speicher (bezogen auf die installierte
+Leistung von PV-Aufdachanlagen), aggregiert für gesamte Region, für folgende
+Randbedingungen:
+
+- Alle PV-Anlagen: `all_storages`
+- PV-Anlagen mit 2..20 kWp sowie Batteriespeicher <20 kWh und <20 kW (kann in
+ [config.yml](../../digipipe/store/datasets/bnetza_mastr_storage_region/config.yml) unter `home_storages` konfiguriert werden):
+ `home_storages`
+
+**Dataset: `datasets/bnetza_mastr_storage_region`**
+
+
+------------------------------
+## EE-Einspeisezeitreihen
+
+Einspeisezeitreihen für Erneuerbare Energien. Als Wetterjahr wird 2011
+verwendet, siehe [Szenarien](../../digipipe/store/../../docs/sections/scenarios.md).
+
+Raw dataset mit methodischer Beschreibung:
+[renewables.ninja_feedin](../../digipipe/store/raw/renewables.ninja_feedin/dataset.md)
+
+### Einspeisezeitreihen
+
+Zeitreihe normiert auf Summe=1 für
+
+- Windenergie: `wind_feedin_timeseries.csv`
+- Photovoltaik: `pv_feedin_timeseries.csv`
+- Solarthermie: `st_feedin_timeseries.csv`
+- Laufwasserkraft: `ror_feedin_timeseries.csv`
+
+**Dataset: `datasets/renewable_feedin`**
+
+
+------------------------------
+## Emissionen
+
+Emissionen für Sachsen-Anhalt und die Region, aggregiert nach Sektoren der
+CRF-Nomenklatur.
+
+Datei `emissions.json` enthält Chartdaten.
+
+Raw dataset: [emissions](../../digipipe/store/raw/emissions/dataset.md)
+
+**Dataset: `datasets/emissions_region`**
+
+
+------------------------------
+## Geo- oder Solarthermie-, Grubengas- und Klärschlamm-Anlagen
+
+Anlagen der Geo- oder Solarthermie, Grubengas und Klärschlamm in der Region
+aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_gsgk_stats_muns.csv`.
+
+**Dataset: `datasets/bnetza_mastr_gsgk_region`**
+
+
+------------------------------
+## Strombedarf
+
+Nettostrombedarfe und -zeitreihen für Haushalte, GHD und Industrie je Gemeinde.
+
+Die Berechnung der regionalen Prognosewerte je Verbrauchssektor erfolgt anhand
+landesweiter Prognosen aus den
+[BMWK Langfristszenarien](../../digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md).
+
+### Haushalte
+
+- Jährlicher Strombedarf je Gemeinde in MWh aus
+ [DemandRegio](../../digipipe/store/preprocessed/demandregio/dataset.md), von Landkreis- auf
+ Gemeindeebene disaggregiert anhand von Bevölkerungsprognosen
+ ([STALA ST](../../digipipe/store/preprocessed/stala_st_pop_prog/dataset.md)).
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion des
+ Strombedarfs (ohne Wärmegewinnung) aus
+ [BMWK Langfristszenarien](../../digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Hierbei wird das Szenario "TN-Strom" als Grundlage für den Status
+ quo verwendet und Werte für 2022 interpoliert. Die Zielwerte werden dem
+ Szenario "T45-Strom" entnommen.
+- Gemittelte, normierte Strombedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../digipipe/store/preprocessed/demandregio/dataset.md)-Daten von 2022, die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird, da die Basis
+ SLP-Profile sind und Differenzen zwischen verschiedenen Jahren nur aufgrund
+ der Lage von Wochenenden und Feiertagen bestehen. Diese werden daher
+ vernachlässigt.
+
+### GHD
+
+- Jährlicher Strombedarf je Gemeinde in MWh aus
+ [DemandRegio](../../digipipe/store/preprocessed/demandregio/dataset.md), von Landkreis- auf
+ Gemeindeebene disaggregiert anhand von sozialversicherungspflichtig
+ Beschäftigten im Jahr 2022
+ ([BA für Arbeit](../../digipipe/store/preprocessed/ba_employment/dataset.md)).
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion des
+ Strombedarfs (ohne Wärmegewinnung) aus
+ [BMWK Langfristszenarien](../../digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Hierbei wird das Szenario "TN-Strom" als Grundlage für den Status
+ quo verwendet und Werte für 2022 interpoliert. Die Zielwerte werden dem
+ Szenario "T45-Strom" entnommen.
+- Gemittelte, normierte Strombedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../digipipe/store/preprocessed/demandregio/dataset.md)-Daten von 2022, die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird. Basis bilden
+ sowohl SLP- als auch branchenspezifische Profile. Aufgrund der geringen
+ Differenzen zwischen den Landkreisen werden diese gemittelt. Differenzen
+ zwischen verschiedenen Jahren bestehen nur aufgrund der Lage von Wochenenden
+ und Feiertagen und werden daher vernachlässigt.
+
+### Industrie
+
+- Jährlicher Strombedarf je Gemeinde in MWh. Hierfür stehen 2 Datensätze zur
+ Verfügung - welcher verwendet wird, kann in der [Konfiguration](../../digipipe/store/datasets/demand_electricity_region/config.yml)
+ via `ind_electricity_demand_source` eingestellt werden:
+ - [DemandRegio](../../digipipe/store/preprocessed/demandregio/dataset.md): Werte für alle
+ Landkreise in Deutschland.
+ - [STALA ST](../../digipipe/store/preprocessed/stala_st_energy/dataset.md) (Standard):
+ Genauere Werte, jedoch nur für Sachsen-Anhalt verfügbar.
+- Die Desaggregation von Landkreis- auf Gemeindeebene erfolgt anhand der
+ Beschäftigten im verarbeitenden Gewerbe im Jahr 2022
+ ([Regionalstatistik](../../digipipe/store/preprocessed/regiostat/dataset.md)).
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion des
+ industriellen Gesamtenergiebedarfs aus
+ [BMWK Langfristszenarien](../../digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Im Unterschied zu Haushalten und GHD liegen die Daten für den
+ Wärme- und Stromanteil nicht getrennt vor, sodass auf den
+ Gesamtenergiebedarf zurückgegriffen wird.
+ Es wird das Szenario "TN-Strom" als Grundlage für den Status quo verwendet und
+ Werte für 2022 interpoliert. Die Zielwerte werden dem Szenario "T45-Strom"
+ entnommen.
+- Gemittelte, normierte Strombedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../digipipe/store/preprocessed/demandregio/dataset.md)-Daten von 2022, die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird. Basis bilden
+ sowohl SLP- als auch branchenspezifische Profile. Aufgrund der geringen
+ Differenzen zwischen den Landkreisen werden diese gemittelt. Differenzen
+ zwischen verschiedenen Jahren bestehen nur aufgrund der Lage von Wochenenden
+ und Feiertagen und werden daher vernachlässigt.
+
+**Dataset: `datasets/demand_electricity_region`**
+
+
+------------------------------
+## Dachflächenpotenzial PV-Aufdachanlagen in ABW
+
+Abschätzung der installierten Leistung und des Ertrags von PV-Aufdachanlagen in
+Anhalt-Bitterfeld-Wittenberg der Regionalen Planungsgemeinschaft aus Datensatz
+[rpg_abw_pv_roof_potential](../../digipipe/store/raw/rpg_abw_pv_roof_potential/dataset.md).
+
+Die Gebäudezentroide werden mit den Gemeindegrenzen verschnitten und den
+Gemeinden zugeordnet.
+Ergebnisdaten:
+
+- Alle Gebäude: `potentialarea_pv_roof_area_stats_muns.csv`
+- Alle nicht denkmalgeschützten Gebäude:
+ `potentialarea_pv_roof_wo_historic_area_stats_muns.csv`
+
+Des Weiteren wird je Gemeinde der relative Anteil der bereits installierten
+Anlagenleistung an der theoretisch installierbaren Leistung (bei
+100% Dachnutzung) berechnet.
+Ergebnisdaten:
+
+- Alle Gebäude: `potentialarea_pv_roof_deployment_stats_muns.csv`
+- Alle nicht denkmalgeschützten Gebäude:
+ `potentialarea_pv_roof_wo_historic_deployment_stats_muns.csv`
+
+Die Gemeinden werden über den Schlüssel `municipality_id` (vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_muns_region/dataset.md))
+identifiziert.
+
+### Ausbauziele
+
+Es werden PV-Ausbauziele für die Region berechnet, indem die Bundesziele aus den
+[BMWK Langfristszenarien](../../digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md)
+i.H.v. 428 GW
+([§4 EEG 2023](https://www.gesetze-im-internet.de/eeg_2014/__4.html): 400 GW)
+anhand der Gebäudegrundflächen disaggregiert werden. Hierzu wird der Anteil der
+Gebäudegrundflächen in der Region an der bundesweiten Gebäudegrundflächen
+berechnet (s. Datensatz [osm_buildings](../../digipipe/store/datasets/osm_buildings/dataset.md)) und die
+Ziele linear skaliert. Da in den o.g. Ausbauzielen nicht zwischen Freiflächen-
+und Aufdach-PV unterschieden wird, wird ein Verhältnis von 50:50 angenommen,
+d.h. bundesweit 214 GW auf Aufdach-PV entfallen.
+
+Der Anteil beträgt 0,62 % und das Leistungsziel damit 1327 MW, s.
+`potentialarea_pv_roof_regionalized_targets.json`.
+
+**Dataset: `datasets/potentialarea_pv_roof_region`**
+
+
+------------------------------
+## Biomasse-/Biogasanlagen
+
+Biomasse-/Biogasanlagen in der Region aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_biomass_stats_muns.csv`.
+
+**Dataset: `datasets/bnetza_mastr_biomass_region`**
+
+
+------------------------------
+## Staat
+
+Staatsgrenze aus Geodaten der Verwaltungsgebiete extrahiert und nach Landmasse
+gefiltert (Geofaktor 4 = "mit Struktur Land").
+
+**Dataset: `datasets/bkg_vg250_state`**
+
+
+------------------------------
+## Wärmepumpen COP
+
+Zeitreihe für die Leistungszahl / Coefficient of performance (COP) für
+Wärmepumpen. Berücksichtigt werden Luftwärmepumpen (ASHP) und Erdwärmepumpen
+(GSHP). Der COP wird mit Hilfe von Zeitreihen der Umgebungstemperatur (ASHP)
+bzw. der Bodentemperatur (GSHP) für jeden Zeitschritt berechnet.
+
+Details zur Berechnungsmethodik können der Dokumentation von
+[oemof.thermal](https://oemof-thermal.readthedocs.io/en/latest/compression_heat_pumps_and_chillers.html)
+entnommen werden.
+
+Annahmen:
+
+- Vorlauftemperatur: 40 °C
+- Gütegrad / Quality grade: 0.4 (nach
+ [VDE](https://www.energiedialog2050.info/wp-content/uploads/simple-file-list/VDE_ST_ETG_Warmemarkt_RZ-web.pdf))
+- Vereisungsverluste bei ASHP: 20 % bei <2 °C
+
+Daraus ergibt sich eine mittlere Jahresarbeitszahl (JAZ) von 3,3 für ASHP und
+4,3 für GSHP, die mit typischen Werten für 2019
+([AEW](https://static.agora-energiewende.de/fileadmin/Projekte/2022/2022-04_DE_Scaling_up_heat_pumps/A-EW_273_Waermepumpen_WEB.pdf))
+übereinstimmen. Für das Zukunftsszenario wird ferner ein Effizienzgewinn durch
+technische Weiterentwicklung von 25 % angenommen
+[ewi](https://www.ewi.uni-koeln.de/cms/wp-content/uploads/2015/12/2014_06_24_ENDBER_P7570_Energiereferenzprognose-GESAMT-FIN-IA.pdf).
+
+Beide separat erstelle Zeitreihen werden anhand der heutigen Marktdurchdringung
+gewichtet und in eine mittlere Zeitreihe für Wärmepumpen überführt. Im Jahr
+2022 wurden 87 % ASHP und 13 % GSHP abgesetzt nach
+[BWP](https://www.waermepumpe.de/fileadmin/user_upload/waermepumpe/05_Presse/01_Pressemitteilungen/BWP_Branchenstudie_2023_DRUCK.pdf),
+über die letzten 10 Jahre beträgt das Verhältnis ca. 80:20.
+Für 2045 wird daher ein Anteil von 80 % ASHP und 20 % GSHP angenommen.
+
+Verwendet Datensätze:
+
+- [dwd_temperature](../../digipipe/store/preprocessed/dwd_temperature/dataset.md)
+
+**Dataset: `datasets/heatpump_cop`**
+
+
+------------------------------
+## Photovoltaik-Freiflächenanlagen
+
+Photovoltaik-Freiflächenanlagen in der Region aus MaStR-Registerdaten als
+Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_pv_ground_stats_muns.csv`.
+
+### Datenkorrektur
+
+Einige Anlagen sind hinsichtlich Ihrer geografischen Lage oder Typs fehlerhaft.
+Anhand des Datensatzes
+[bnetza_mastr_correction_region](../../digipipe/store/raw/bnetza_mastr_correction_region/dataset.md)
+wird für diese Anlagen eine Datenkorrektur vorgenommen.
+
+**Dataset: `datasets/bnetza_mastr_pv_ground_region`**
+
+
+------------------------------
+## Landkreise
+
+Landkreise der Region aus Geodaten der Verwaltungsgebiete extrahiert und
+nach Landmasse gefiltert (Geofaktor 4 = "mit Struktur Land").
+
+**Dataset: `datasets/bkg_vg250_districts_region`**
+
+
+------------------------------
+## Potenzialgebiete PV-Freiflächen
+
+### Potenzialflächen
+
+Potenzialgebiete für die Errichtung von PV-Freiflächenanlagen aus dem
+[PV- und Windflächenrechner](https://www.agora-energiewende.de/service/pv-und-windflaechenrechner/)
+(s. Datensatz [rli_pv_wfr](../../digipipe/store/raw/rli_pv_wfr/dataset.md)).
+
+Die Potenzialflächen bilden jene Flächen ab, die für die Nutzung durch
+Freiflächen-Photovoltaikanlagen grundsätzlich zur Verfügung stehen. Sie
+orientieren sich an der aktuellen Förderkulisse und wurden anhand des
+Flächenumfangs sowie den verfügbaren Geodaten ausgewählt: Von den in §37 EEG
+2021 definierten Flächen werden Flächen nach §37 Absatz 1 Nummer 2 Buchstaben c,
+h und i berücksichtigt (für Details zur Methodik siehe
+[methodisches Begleitdokument](https://zenodo.org/record/6794558) zum PV- und
+Windflächenrechner).
+
+Dateien:
+
+- Freiflächen-PV auf Acker- und Grünlandflächen mit geringer Bodengüte (Soil
+ Quality Rating (SQR) < 40): `potentialarea_pv_agriculture_lfa-off_region.gpkg`
+- Potenzialflächen für Freiflächen-PV entlang von Bundesautobahnen und
+ Schienenwegen (500m-Streifen): `potentialarea_pv_road_railway_region.gpkg`
+
+### Statistische Auswertung
+
+Die Flächen werden mit den Gemeindegrenzen verschnitten und den Gemeinden
+zugeordnet. Je Gemeinde und obigem Flächentyp/Datei wird eine Flächensumme (in
+km²) berechnet, siehe `potentialarea_pv_ground_area_stats_muns.csv`. Die
+Gemeinden werden über den Schlüssel `municipality_id` (vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_muns_region/dataset.md))
+identifiziert.
+
+Des Weiteren werden die Flächenanteile der verfügbaren Potenzialgebiete - deren
+Nutzung nur eingeschränkt möglich ist (z.B. durch Naturschutzgebieten etc.) -
+gegenüber den gesamten Potenzialgebiete (für die Parametrierung der Regler) nach
+`potentialarea_pv_ground_area_shares.json` exportiert.
+
+### Ausbauziele
+
+Es werden PV-Ausbauziele für die Region berechnet, indem die Bundesziele aus den
+[BMWK Langfristszenarien](../../digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md)
+i.H.v. 428 GW
+([§4 EEG 2023](https://www.gesetze-im-internet.de/eeg_2014/__4.html): 400 GW)
+anhand der regional verfügbaren Potenzialflächen disaggregiert werden. Hierzu
+wird der Anteil der Flächensumme der beiden o.g. Flächentypen an den bundesweit
+verfügbaren Flächen (Datensatz [rli_pv_wfr](../../digipipe/store/raw/rli_pv_wfr/dataset.md))
+berechnet. Da in den o.g. Ausbauzielen nicht zwischen Freiflächen- und
+Aufdach-PV unterschieden wird, wird ein Verhältnis von 50:50 angenommen, d.h.
+bundesweit 214 GW auf Freiflächen-PV entfallen.
+
+Es ergeben sich folgende Flächen- und Leistungsanteile:
+
+Gesamt: 0.38 % (819 MW)
+
+- Entlang von BAB und Schienenwegen: 0.13 % (278 MW)
+- Acker- und Grünlandflächen mit geringer Bodengüte: 0.25 % (541 MW)
+
+Ergebnisse in `potentialarea_pv_ground_regionalized_targets.json`
+
+**Dataset: `datasets/potentialarea_pv_ground_region`**
+
+
+------------------------------
+## Sozialversicherungspflichtig Beschäftigte und Betriebe
+
+Gesamtanzahl sozialversicherungspflichtig Beschäftigte und Betriebsstätten
+je Gemeinde für die Region.
+
+Raw datasets:
+[ba_employment](../../digipipe/store/raw/ba_employment/dataset.md),
+[regiostat](../../digipipe/store/raw/regiostat/dataset.md)
+
+**Dataset: `datasets/employment_region`**
+
+
+------------------------------
+## Bevölkerungsentwicklung
+
+EinwohnerInnen je Gemeinde: Historische Daten und Prognosen
+
+### Historische Daten bis 2022
+
+Statistisches Bundesamt (Raw dataset:
+[destatis_gv](../../digipipe/store/raw/destatis_gv/dataset.md))
+
+### Prognosen bis 2035
+
+Statistisches Landesamt Sachsen-Anhalt (Raw dataset:
+[stala_st_pop_prog](../../digipipe/store/raw/stala_st_pop_prog/dataset.md)). Deaktivieren
+mittels entfernen der Zieljahre in [config.yml](../../digipipe/store/datasets/population_region/config.yml) im Abschnitt
+`prognosis_fstate_munlevel`.
+
+Kann für andere Regionen auch durch DemandRegio (s.u.) ersetzt werden, die
+tatsächliche regionale Auflösung wird dadurch reduziert.
+
+### Prognosen bis 2045
+
+DemandRegio (Raw dataset: [demandregio](../../digipipe/store/raw/demandregio/dataset.md))
+basierend auf der
+[14. koordinierten Bevölkerungsvorausberechnung](https://www.destatis.de/DE/Themen/Gesellschaft-Umwelt/Bevoelkerung/Bevoelkerungsvorausberechnung/aktualisierung-bevoelkerungsvorausberechnung.html)
+der Statistischen Ämter von Bund und Ländern. Diese Daten liegen auf
+Landkreisebene vor, daher erfolgt eine gleichmäßige Skalierung der
+dazugehörigen Gemeinden auf den jeweiligen Prognosewert.
+
+Deaktivieren mittels entfernen der Zieljahre in [config.yml](../../digipipe/store/datasets/population_region/config.yml) im
+Abschnitt `prognosis_germany_districtlevel`.
+
+### Extrapolation
+
+Über 2045 hinaus wird lineare Extrapolation auf Basis der letzten beiden
+Prognosejahre unterstützt. Um diese zu aktivieren, müssen lediglich Zieljahre
+in die [config.yml](../../digipipe/store/datasets/population_region/config.yml) im Abschnitt `extrapolation` eingetragen werden.
+
+**Dataset: `datasets/population_region`**
+
+
+------------------------------
+## Windenergieanlagen
+
+Windenergieanlagen in der Region aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_wind_stats_muns.csv`.
+
+**Dataset: `datasets/bnetza_mastr_wind_region`**
+
+
+------------------------------
+## Bundesländer
+
+Bundesländergrenzen aus Geodaten der Verwaltungsgebiete extrahiert und nach
+Landmasse gefiltert (Geofaktor 4 = "mit Struktur Land").
+
+**Dataset: `datasets/bkg_vg250_federal_states`**
+
+
+------------------------------
+## Photovoltaik-Aufdachanlagen
+
+Photovoltaik-Aufdachanlagen in der Region aus MaStR-Registerdaten als
+Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_pv_roof_stats_muns.csv`.
+
+### Datenkorrektur
+
+Einige Anlagen sind hinsichtlich Ihrer geografischen Lage oder Typs fehlerhaft.
+Anhand des Datensatzes
+[bnetza_mastr_correction_region](../../digipipe/store/raw/bnetza_mastr_correction_region/dataset.md)
+wird für diese Anlagen eine Datenkorrektur vorgenommen.
+
+**Dataset: `datasets/bnetza_mastr_pv_roof_region`**
+
+
+------------------------------
+## Wasserkraftanlagen
+
+Wasserkraftanlagen in der Region aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_hydro_stats_muns.csv`.
+
+**Dataset: `datasets/bnetza_mastr_hydro_region`**
+
+
+------------------------------
+## Wärmebedarf
+
+Wärmebedarfe (Endenergie) Fernwärme und dezentrale Wärme sowie Wärmezeitreihen
+für Haushalte, GHD und Industrie je Gemeinde.
+
+### Gesamtwärmebedarf
+
+Die Berechnung der regionalen Prognosewerte je Verbrauchssektor erfolgt anhand
+landesweiter Prognosen aus den
+[BMWK Langfristszenarien](../../digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md).
+
+#### Haushalte
+
+- Jährlicher Wärmebedarf je Gemeinde in MWh: Bundeswert aus
+ [AG Energiebilanzen](../../digipipe/store/preprocessed/ageb_energy_balance/dataset.md)
+ 2021 für Raumwärme, Warmwasser und Prozesswärme, desaggregiert auf Gemeinden
+ mittels Wärmebedarfs-Rasterdaten aus 2015 (Wärmebedarfsdichte 1ha) aus
+ [Peta5](../../digipipe/store/raw/seenergies_peta5/dataset.md).
+ Anm.: Die Desaggregation könnte alternativ über Zensus "Gebäude mit Wohnraum
+ nach Heizungsart" (31231-02-01-5, s.
+ [regiostat](../../digipipe/store/raw/regiostat/dataset.md) erfolgen)
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion der
+ Gebäudewärmebedarfe aus
+ [BMWK Langfristszenarien](../../digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Hierbei wird das Szenario "TN-Strom" als Grundlage für den Status
+ quo verwendet und Werte für 2022 interpoliert. Die Zielwerte werden dem
+ Szenario "T45-Strom" entnommen.
+- Gemittelte, normierte Gasbedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../digipipe/store/preprocessed/demandregio/dataset.md)-Daten von 2022 die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird, da die Basis
+ SLP-Profile sind und Differenzen zwischen verschiedenen Jahren nur aufgrund
+ der Lage von Wochenenden und Feiertagen bestehen. Diese werden daher
+ vernachlässigt.
+
+#### GHD
+
+- Jährlicher Wärmebedarf je Gemeinde in MWh: Bundeswert aus
+ [AG Energiebilanzen](../../digipipe/store/preprocessed/ageb_energy_balance/dataset.md)
+ 2021 für Raumwärme, Warmwasser und Prozesswärme, desaggregiert auf Gemeinden
+ mittels Wärmebedarfs-Rasterdaten aus 2015 (Wärmebedarfsdichte 1ha) aus
+ [Peta5](../../digipipe/store/raw/seenergies_peta5/dataset.md)
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion der
+ Gebäudewärmebedarfe aus
+ [BMWK Langfristszenarien](../../digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Hierbei wird das Szenario "TN-Strom" als Grundlage für den Status
+ quo verwendet und Werte für 2022 interpoliert. Die Zielwerte werden dem
+ Szenario "T45-Strom" entnommen.
+- Gemittelte, normierte Gasbedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../digipipe/store/preprocessed/demandregio/dataset.md)-Daten von 2022 die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird, da die Basis
+ SLP-Profile sind und Differenzen zwischen verschiedenen Jahren nur aufgrund
+ der Lage von Wochenenden und Feiertagen bestehen. Diese werden daher
+ vernachlässigt.
+
+#### Industrie
+
+- Jährlicher Wärmebedarf je Gemeinde in MWh: Bundeswert aus
+ [AG Energiebilanzen](../../digipipe/store/preprocessed/ageb_energy_balance/dataset.md)
+ 2021 für Raumwärme, Warmwasser und Prozesswärme. Die Desaggregation auf
+ Landkreisebene erfolgt anhand des Gesamtenergiebedarfs im verarbeitenden
+ Gewerbe aus [Regionalstatistik](../../digipipe/store/preprocessed/regiostat/dataset.md).
+ Die anschließende Desaggregation auf Gemeindeebene wird mittels
+ Beschäftigtenzahlen im verarbeitenden Gewerbe in 2022 aus
+ [Regionalstatistik](../../digipipe/store/preprocessed/regiostat/dataset.md) vorgenommen.
+- Prognosewerte für 2045 werden durch lineare Skalierung mittels Reduktion des
+ industriellen Gesamtenergiebedarfs aus
+ [BMWK Langfristszenarien](../../digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md)
+ berechnet. Im Unterschied zu Haushalten und GHD liegen die Daten für den
+ Wärme- und Stromanteil nicht getrennt vor, sodass auf den
+ Gesamtenergiebedarf zurückgegriffen wird.
+ Es wird das Szenario "TN-Strom" als Grundlage für den Status quo verwendet und
+ Werte für 2022 interpoliert. Die Zielwerte werden dem Szenario "T45-Strom"
+ entnommen.
+- Gemittelte, normierte Gasbedarfszeitreihe (auf 1 MWh) aus
+ [DemandRegio](../../digipipe/store/preprocessed/demandregio/dataset.md)-Daten von 2022 die
+ für alle Zielszenarien und Aggregationsebenen verwendet wird, da die Basis
+ SLP-Profile sind und Differenzen zwischen verschiedenen Jahren nur aufgrund
+ der Lage von Wochenenden und Feiertagen bestehen. Diese werden daher
+ vernachlässigt.
+- Es erfolgt keine Aufteilung des Wärmebedarfs auf unterschiedliche
+ Temperaturniveaus.
+
+### Dezentrale Wärme und Fernwärme
+
+Der Gesamtwärmebedarf wird auf dezentrale Heizsysteme und Fernwärme aufgeteilt.
+Fernwärmenetze existieren in Dessau-Roßlau, Bitterfeld-Wolfen, Köthen und
+Wittenberg.
+
+Da keine Daten zum tatsächlichen Fernwärmebedarf vorliegen, werden Annahmen auf
+Basis folgender Quellen getroffen:
+
+- [Zensus 2011: Gebäude nach Heizungsart](https://www.regionalstatistik.de/genesis//online?operation=table&code=31211-04-01-5-B)
+- [BMWK Langfristszenarien: Wärmenachfrage in Wärmenetzen (HH&GHD) (2025)](https://enertile-explorer.isi.fraunhofer.de:8443/open-view/54022/62a2667df6f8c176ff129f7ede944837)
+- [STALA ST: Wärmebilanz der Industriebetriebe (2021)](https://statistik.sachsen-anhalt.de/themen/wirtschaftsbereiche/energie-und-wasserversorgung/tabellen-energieverwendung#c256237)
+- [STALA ST: Energie- und Wasserversorgung](https://statistik.sachsen-anhalt.de/fileadmin/Bibliothek/Landesaemter/StaLa/startseite/Themen/Energie/Berichte/6E403_2020-A.pdf)
+- [WindNODE](https://windnode-abw.readthedocs.io/en/latest/energy_system_model.html#district-heating)
+- [Peta5: D5 1 District Heating Areas (2020)](https://s-eenergies-open-data-euf.hub.arcgis.com/datasets/b62b8ad79f0e4ae38f032ad6aadb91a0_0/)
+
+Annahmen zu Fernwärmeanteilen (Anteil der Endenergie aus Fernwärme an gesamter
+Wärme-Endenergie) je Bedarfssektor:
+
+| Fernwärmenetz | Haushalte | GHD | Industrie |
+|-------------------|----------:|-----:|----------:|
+| Dessau-Roßlau | 0,36 | 0,36 | 0,19 |
+| Bitterfeld-Wolfen | 0,11 | 0,11 | 0,21 |
+| Köthen | 0,07 | 0,07 | 0,21 |
+| Wittenberg | 0,15 | 0,15 | 0,01 |
+
+Die Fernwärmeanteile können in der [config.yml](../../digipipe/store/datasets/demand_heat_region/config.yml) im Abschnitt
+`district_heating_share` für jeden Sektor separat angepasst werden. Es wird
+vereinfachend angenommen, dass der Anteil an Fernwärme für alle
+Szenarien/Zieljahre gleich bleibt.
+
+### Beheizungsstruktur
+
+Die Beheizungsstruktur für 2020 und 2045 wird den
+[BMWK Langfristszenarien](../../digipipe/store/preprocessed/bmwk_long_term_scenarios/dataset.md)
+entnommen (Gebäude: Haushalte und GHD Energiebedarf) und für 2022 interpoliert.
+Hierbei wird nach Technologien für dezentrale sowie Fernwärme unterschieden.
+Für die Biomasse wird der relative Energiebedarf mit Hilfe von Anteilen der
+installierten Leistung von spezifischen Biomasse-Konversionsanlagen
+[dbfz_biomasss_capacity_rel](../../digipipe/store/preprocessed/dbfz_biomass_capacity_rel/dataset.md)
+je Technologie aufgelöst. Der Vereinfachung halber wird angenommen, dass die
+relative installierte Leistung der relativen Energiemenge entspricht.
+
+### Ergebnisdaten
+
+- Haushalte: Wärmebedarf gesamt: `demand_hh_heat_demand.csv`
+- Haushalte: Wärmebedarf Fernwärme: `demand_hh_heat_demand_cen.csv`
+- Haushalte: Wärmebedarf dezentrale Wärme: `demand_hh_heat_demand_dec.csv`
+- Haushalte: Zeitreihen: `demand_hh_heat_timeseries.csv`
+
+- GHD: Wärmebedarf gesamt: `demand_cts_heat_demand.csv`
+- GHD: Wärmebedarf Fernwärme: `demand_cts_heat_demand_cen.csv`
+- GHD: Wärmebedarf dezentrale Wärme: `demand_cts_heat_demand_dec.csv`
+- GHD: Zeitreihen: `demand_cts_heat_timeseries.csv`
+
+- Industrie: Wärmebedarf gesamt: `demand_ind_heat_demand.csv`
+- Industrie: Wärmebedarf Fernwärme: `demand_ind_heat_demand_cen.csv`
+- Industrie: Wärmebedarf dezentrale Wärme: `demand_ind_heat_demand_dec.csv`
+- GHD: Zeitreihen: `demand_ind_heat_timeseries.csv`
+
+- Beheizungsstruktur dezentral (informativ): `demand_heat_structure_dec.csv`
+- Beheizungsstruktur zentral (informativ): `demand_heat_structure_cen.csv`
+- Beheizungsstruktur dezentral für Weiterverwendung im Energiesystem:
+ `demand_heat_structure_esys_dec.csv`
+- Beheizungsstruktur Fernwärme für Weiterverwendung im Energiesystem:
+ `demand_heat_structure_esys_cen.csv`
+
+**Dataset: `datasets/demand_heat_region`**
+
+
+------------------------------
+## OpenStreetMap Gebäude
+
+OSM Gebäude aus [osm_filtered](../../digipipe/store/preprocessed/osm_filtered/dataset.md)
+mittels OGR extrahieren und nach Tags (s. [config.yml](../../digipipe/store/datasets/osm_buildings/config.yml)) filtern.
+
+Ziel ist die Ermittlung des regionalen Anteils Gebäudegrundflächen an der
+gesamten Gebäudegrundfläche in Deutschland.
+
+Schritte:
+
+- Extraktion aller Gebäude in Deutschland --> `osm_buildings.gpkg`
+- Zentroide und Fläche je Gebäude erstellen --> `osm_buildings_centroids.gpkg`
+- Mit Region verschneiden --> `osm_buildings_centroids_region.gpkg`
+- Flächensumme berechnen --> `osm_buildings_ground_area_region.gpkg`,
+ `osm_buildings_ground_area_country.gpkg`
+- Regionalen Anteil berechnen --> `osm_buildings_ground_area_share_region.json`
+
+**Achtung:** Konvertierungs- und Extraktionsprozess benötigt ~15 GB
+Speicherplatz und kann viel Zeit in Anspruch nehmen.
+
+**Dataset: `datasets/osm_buildings`**
+
+
+------------------------------
+## Verbrennungskraftwerke
+
+Verbrennungskraftwerke in der Region aus MaStR-Registerdaten als Geopackage.
+Es werden alle Anlagen berücksichtigt, die in Betrieb sind oder sich in
+Planung befinden. Anlagen mit Geokoordinaten werden georeferenziert
+übernommen, für Anlagen die keine Koordinaten aufweisen (üblicherweise <=30
+kW Nennleistung) erfolgt ein Geocoding anhand von PLZ und Ort, um eine
+ungefähre Position bereit zu stellen.
+
+Neben einem anlagenscharfen Datensatz wird ein weiterer Datensatz erzeugt,
+der alle Anlagen mit approximierter Position je Position zusammenfasst und
+jeweils typische Kennwerte enthält (u.a. Anzahl Anlagen, Gesamtleistung).
+
+Jede Anlage wird anhand ihrer Lokation einer Gemeinde (Attribut
+`municipality_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_muns_region/dataset.md)) und
+einem Landkreis (Attribut `district_id`, vgl.
+[bkg_vg250_muns_region](../../digipipe/store/datasets/bkg_vg250_districts_region/dataset.md))
+zugeordnet.
+
+Zusätzlich erfolgt eine statistische Auswertung der installierten Leistung in
+`bnetza_mastr_combustion_stats_muns.csv`.
+
+**Dataset: `datasets/bnetza_mastr_combustion_region`**
+
+
+------------------------------
+## Gemeinden
+
+Gemeinden der Region aus Geodaten der Verwaltungsgebiete extrahiert und
+nach Landmasse gefiltert (Geofaktor 4 = "mit Struktur Land").
+
+**Dataset: `datasets/bkg_vg250_muns_region`**
+
+
+------------------------------
+## Bezeichner und Namen aus MaStR
+
+Bezeichner und Namen aus MaStR als Mapping ->
+ wobei CamelCase aus in Leerzeichen konvertiert
+werden.
+
+**Dataset: `datasets/bnetza_mastr_captions`**
+
+
+------------------------------
+## Geodaten PV- und Windflächenrechner
+
+Geodaten aus dem [PV- und Windflächenrechner](https://www.agora-energiewende.de/service/pv-und-windflaechenrechner/),
+extrahiert, zu LAEA Europe (EPSG:3035) umprojiziert und auf die Regionsgrenzen
+zugeschnitten.
+
+Preprocessed dataset:
+[rli_pv_windflaechenrechner](../../digipipe/store/preprocessed/rli_pv_wfr/dataset.md)
+
+**Dataset: `datasets/rli_pv_wfr_region`**
+
+
+------------------------------
+## Region
+
+Region aus Geodaten der Landkreise zusammengeführt.
+
+**Dataset: `datasets/bkg_vg250_region`**
+
+
+------------------------------
+## Settings für App
+
+Einstellungen für die App.
+
+### Layerliste (rechtes Panel)
+
+- Konfiguration: [config.yml](https://github.com/rl-institut/digipipe/blob/main/digipipe/store/datasets/app_settings/config.yml) --> `map_panel_layer_list`
+- Ergebnisfile: `map_panel_layer_list.json`
+- Wird manuell in die App eingepflegt (s.
+ [map_config.py](https://github.com/rl-institut/digiplan/blob/main/digiplan/map/map_config.py))
+
+### Settings panels
+
+Die im linken Panel aufgeführten Einstellelemente (Slider und Schalter) werden
+hier parametriert.
+
+- Konfiguration des Templates:
+ [config.yml](https://github.com/rl-institut/digipipe/blob/main/digipipe/store/datasets/app_settings/config.yml) --> `panel_settings_templates`
+- Parametrierung der Slider und Schalter:
+ [panels.py](https://github.com/rl-institut/digipipe/blob/main/digipipe/store/datasets/app_settings/scripts/panels.py)
+- Ergebnisfiles:
+ - `energy_settings_panel.json`
+ - `heat_settings_panel.json`
+ - `traffic_settings_panel.json`
+- Werden in die App eingelesen
+
+#### Parametrierung der Einstellelemente
+
+Für die Slider werden folgende Attribute gesetzt:
+Minimum, Maximum, Schrittweite, Startwert, Status-quo-Wert, Zielwert 2045.
+Diese werden wie folgt bestimmt (vgl. auch (i)-Tooltips an den Elementen):
+
+| **Technologie** | **Element id** | **Maximum** | **Startwert** | **Status-quo-Wert** | **Zielwert 2045** |
+|-------------------------|----------------|-------------------------------------|---------------------|-------------------------------------------------|------------------------------------------------|
+| Windenergie | `s_w_1` | Inst. Leistung in bestehenden VR/EG | Wie Status-quo-Wert | Inst. Leistung 2022 | Aus Flächenziel Sachsen-Anhalt (2,2 % in 2032) |
+| | `s_w_3` | - | Wie Status-quo-Wert | On | - |
+| | `s_w_4` | - | Wie Status-quo-Wert | Off | - |
+| | `s_w_4_1` | - | Wie Status-quo-Wert | On | - |
+| | `s_w_4_2` | - | Wie Status-quo-Wert | Off | - |
+| | `s_w_5` | - | Wie Status-quo-Wert | Off | - |
+| | `s_w_5_1` | 100 % | Wie Status-quo-Wert | Theoret. Wert berechnet aus inst. Leistung 2022 | - |
+| | `s_w_5_2` | 100 % | Wie Status-quo-Wert | Theoret. Wert berechnet aus inst. Leistung 2022 | - |
+| Freiflächen-PV | `s_pv_ff_1` | | Wie Status-quo-Wert | Inst. Leistung 2022 | Aus EEG 2023 und regionalen Potenzialen |
+| | `s_pv_ff_3` | 100 % | Wie Status-quo-Wert | Theoret. Wert berechnet aus inst. Leistung 2022 | - |
+| | `s_pv_ff_4` | 100 % | Wie Status-quo-Wert | Theoret. Wert berechnet aus inst. Leistung 2022 | - |
+| Aufdach-PV | `s_pv_d_1` | | Wie Status-quo-Wert | Inst. Leistung 2022 | Aus EEG 2023 und regionalen Potenzialen |
+| | `s_pv_d_3` | 100 % | Wie Status-quo-Wert | Theoret. Wert berechnet aus inst. Leistung 2022 | - |
+| | `s_pv_d_4` | 100 % | Wie Status-quo-Wert | Aus MaStR | - |
+| Wasserkraft | `s_h_1` | Inst. Leistung 2022 | Wie Status-quo-Wert | Inst. Leistung 2022 | Inst. Leistung 2022 |
+| Stromverbrauch | `s_v_1` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `s_v_3` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `s_v_4` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `s_v_5` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| Batterie-Großspeicher | `s_s_g_1` | 50 % | Wie Status-quo-Wert | Aus inst. Kapazität und Einspeisung 2022 | - |
+| | | | Wie Status-quo-Wert | | |
+| WP dezentral | `w_d_wp_1` | 95 % | 50 % | Inst. Leistung 2022 aus BMWK Langfristszenarien | Wert 2045 aus BMWK Langfristszenarien |
+| | `w_d_wp_3` | 95 % | 50 % | - | - |
+| | `w_d_wp_4` | 95 % | 50 % | - | - |
+| | `w_d_wp_5` | 95 % | 50 % | - | - |
+| WP zentral | `w_z_wp_1` | 95 % | 50 % | Inst. Leistung 2022 aus BMWK Langfristszenarien | Wert 2045 aus BMWK Langfristszenarien |
+| Wärmeverbrauch | `w_v_1` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `w_v_3` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `w_v_4` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| | `w_v_5` | 200 % | Wie Status-quo-Wert | Verbrauch 2022 (100 %) | Wert 2045 aus BMWK Langfristszenarien |
+| Wärmespeicher dezentral | `w_d_s_1` | 200 % | 100 % | - | - |
+| Wärmespeicher zentral | `w_z_s_1` | 200 % | 100 % | - | - |
+
+Die Maxima der Regler im Hauptpanel (`s_w_1`, `s_pv_ff_1` usw.) werden in der
+App dynamisch aus den durch die UserInnen vorgenommenen Detaileinstellungen
+(`s_w_3`, `s_pv_ff_1` usw.) berechnet.
+
+**Dataset: `datasets/app_settings`**
+
+
+------------------------------
+## Captions
+
+Beschriftungen für WebApp.
+
+Dateien:
+
+- Felder: `captions_fields.json`
+
+**Dataset: `datasets/app_captions`**
+
diff --git a/docs/datasets/preprocessed_datasets.md b/docs/datasets/preprocessed_datasets.md
new file mode 100644
index 00000000..e1a22d85
--- /dev/null
+++ b/docs/datasets/preprocessed_datasets.md
@@ -0,0 +1,243 @@
+# 'Preprocessed' Datasets
+
+------------------------------
+## Lokale Verwaltungseinheiten
+
+Lokale Verwaltungseinheiten (LAUs) von Eurostat, mit NUTS kompatibel. Diese
+LAUs sind die Bausteine der NUTS und umfassen die Gemeinden und Kommunen der
+Europäischen Union.
+
+Daten aus Excel extrahiert und in CSV exportiert.
+
+**Dataset: `preprocessed/eurostat_lau`**
+
+
+------------------------------
+## Bevölkerungsprognose Sachsen-Anhalt
+
+Bevölkerungsprognose je Gemeinde bis 2035 des Statistischen Landesamtes
+Sachsen-Anhalt, extrahiert und konvertiert.
+
+Raw dataset:
+[stala_st_pop_prog](../../digipipe/store/raw/stala_st_pop_prog/dataset.md)
+
+**Dataset: `preprocessed/stala_st_pop_prog`**
+
+
+------------------------------
+## Dachflächenpotenzial PV-Aufdachanlagen in ABW
+
+Abschätzung der installierten Leistung und des Ertrags von PV-Aufdachanlagen in
+Anhalt-Bitterfeld-Wittenberg der Regionalen Planungsgemeinschaft, reprojizert.
+
+Raw dataset:
+[rpg_abw_pv_roof_potential](../../digipipe/store/raw/rpg_abw_pv_roof_potential/dataset.md)
+
+**Dataset: `preprocessed/rpg_abw_pv_roof_potential`**
+
+
+------------------------------
+## Temperatur
+
+Stündliche Mittelwerte der Luft- und Erdbodentemperatur für die Region ABW,
+Mittelwert für alle Gemeinden.
+
+Verwendet: [dwd_temperature](../../digipipe/store/raw/dwd_temperature/dataset.md)
+
+**Dataset: `preprocessed/dwd_temperature`**
+
+
+------------------------------
+## BMWK Langfristszenarien
+
+Langfristszenarien des Bundesministerium für Wirtschaft und Klimaschutz, Daten
+auf Landesebene, extrahiert.
+
+Raw dataset:
+[bmwk_long_term_scenarios](../../digipipe/store/raw/bmwk_long_term_scenarios/dataset.md)
+
+**Dataset: `preprocessed/bmwk_long_term_scenarios`**
+
+
+------------------------------
+## AGEB – Anwendungsbilanzen für die Endenergiesektoren 2011 bis 2021
+
+Detaillierte Anwendungsbilanzen der Endenergiesektoren für 2020 und 2021 sowie
+zusammenfassende Zeitreihen zum Endenergieverbrauch nach Energieträgern und
+Anwendungszwecken für Jahre von 2011 bis 2021 der AG Energiebilanzen.
+
+Aus PDF extrahierte Tabellenwerte für Haushalte, GHD und Industrie.
+
+**Dataset: `preprocessed/ageb_energy_balance`**
+
+
+------------------------------
+## sEEnergies Pan-European Thermal Atlas 5.2 (Peta5)
+
+Wärmebedarf (extrahiert) für Europa 2015 in GJ (1ha Auflösung) für
+
+- Haushalte: Raumwärme und Warmwasser
+- GHD: Raumwärme, Warmwasser und Prozesswärme
+
+**Dataset: `preprocessed/seenergies_peta5`**
+
+
+------------------------------
+## OpenStreetMap gefiltert
+
+OSM data nach bestimmten Tags (s. [config.yml](../../digipipe/store/preprocessed/osm_filtered/config.yml) --> `tags`) gefiltert,
+zu LAEA Europe (EPSG:3035) umprojiziert und in ein Geopackage konvertiert.
+
+**Achtung:** Konvertierungs- und Extraktionsprozess benötigt ~50 GB
+Speicherplatz und kann viel Zeit in Anspruch nehmen.
+
+**Dataset: `preprocessed/osm_filtered`**
+
+
+------------------------------
+## Sozialversicherungspflichtig Beschäftigte und Betriebe
+
+Gemeindedaten der sozialversicherungspflichtig Beschäftigten am 30.06.2022 nach
+Wohn- und Arbeitsort - Deutschland, Länder, Kreise und Gemeinden (Jahreszahlen)
+der Bundesagentur für Arbeit.
+Anzahl Beschäftigte und Betriebe extrahiert und in CSV konvertiert.
+
+**Dataset: `preprocessed/ba_employment`**
+
+
+------------------------------
+## Bevölkerung
+
+Einwohnerzahl nach Gemeinden des Statistischen Bundesamts für die Jahre
+2010, 2015, 2020, 2021, 2022.
+
+**Dataset: `preprocessed/destatis_gv`**
+
+
+------------------------------
+## Regionalstatistik (GENESIS)
+
+Enthält Datensätze der statistischen Ämter des Bundes und der Länder aus
+[regiostat](../../digipipe/store/raw/regiostat/dataset.md).
+
+### Energieverwendung der Betriebe im Verarbeitenden Gewerbe (43531-01-02-4)
+
+Jahreserhebung ü. die Energieverwendung der Betriebe im verarbeitendem Gewerbe.
+
+Änderungen:
+
+- Dateiformat konvertiert
+- Bundesland-, Kreis und Gemeindewerte extrahiert
+- Energie in TWh konvertiert
+
+### Betriebe, tätige Personen, Bruttoentgelte (42111-01-04-5)
+
+Jahreserhebung ü. Betriebe, tätige Personen und Bruttoentgelte der Betriebe im
+verarbeitendem Gewerbe.
+
+Änderungen:
+
+- Dateiformat konvertiert
+- Bundesland-, Kreis und Gemeindewerte extrahiert
+
+**Dataset: `preprocessed/regiostat`**
+
+
+------------------------------
+## Geodaten PV- und Windflächenrechner
+
+Geodaten aus dem [PV- und Windflächenrechner](https://www.agora-energiewende.de/service/pv-und-windflaechenrechner/),
+extrahiert.
+
+Raw dataset:
+[rli_pv_windflaechenrechner](../../digipipe/store/raw/rli_pv_wfr/dataset.md)
+
+**Dataset: `preprocessed/rli_pv_wfr`**
+
+
+------------------------------
+## Regionalplan Anhalt-Bitterfeld-Wittenberg
+
+Vorverarbeitete Datensätze aus Teilplänen Wind der Regionalen
+Planungsgemeinschaft Anhalt-Bitterfeld-Wittenberg aus
+[rpg_abw_regional_plan](../../digipipe/store/raw/rpg_abw_regional_plan/dataset.md).
+
+In der [config.yml](../../digipipe/store/preprocessed/rpg_abw_regional_plan/config.yml) können Einstellungen vorgenommen werden.
+
+**Dataset: `preprocessed/rpg_abw_regional_plan`**
+
+
+------------------------------
+## Erzeugungsanlagen aus Marktstammdatenregister
+
+Erzeugungsanlagen aus dem MaStR für ausgewählte Technologien.
+
+**Dataset: `preprocessed/bnetza_mastr`**
+
+
+------------------------------
+## Energiedaten Sachsen-Anhalt
+
+Datensätze zur Energie- und Wasserversorgung des Statistischen Landesamtes
+Sachsen-Anhalt, extrahiert und konvertiert.
+
+### Daten
+
+Stromverbrauch der Industriebetriebe nach Kreisen 2003-2021 in MWh
+
+- Datei: `power_demand_industry_st_districts.csv`
+
+Raw dataset:
+[stala_st_energy](../../digipipe/store/raw/stala_st_energy/dataset.md)
+
+**Dataset: `preprocessed/stala_st_energy`**
+
+??? metadata "Metadata"
+ ```json
+ {
+ "Datenquellen": {
+ "Stromverbrauch der Industriebetriebe nach Kreisen": "https://statistik.sachsen-anhalt.de/themen/wirtschaftsbereiche/energie-und-wasserversorgung/tabellen-energieverwendung#c206986"
+ }
+ }
+ ```
+
+------------------------------
+## DemandRegio
+
+Regionalisierte Bevölkerungsprognose sowie Strom-, Wärme und Gasbedarf auf
+Landkreisebene, extrahiert.
+
+Enthält Jahresverbräuche und Zeitreihen für die Sektoren Haushalte, Gewerbe,
+Handel, Dienstleistungen (GHD) und Industrie für mehrere Zieljahre.
+
+**Dataset: `preprocessed/demandregio`**
+
+
+------------------------------
+## Anteile von Biomasse-Konversionsanlagen anhand installierter Leistung
+
+Berechnung der Anteile der installierten Leistung an der gesamten installierten
+Leistung der Biomasse-Konversionsanlagen.
+
+Die installierten Leistungen werden
+[dbfz_biomass_heat_capacities](../../digipipe/store/raw/dbfz_biomass_heat_capacities/dataset.md)
+entnommen. Sie werden nach Energieträger (Biogas, Methan oder Holz) und
+Technologie (BHKW (bpchp), Turbine mit Kondensationsentnahme (extchp) oder
+Ofen (oven)) zusammengefasst. Anschließend wird der Anteil der installierten
+Leistung an der gesamten installierten Leistung der Biomasse-Konversionsanlagen
+berechnet. Der Einfachheit halber werden die Projektionen für 2050 dem Jahr
+2045 und die für 2020 dem Jahr 2022 zugeordnet. Der Energieträger und die
+Technologie (vgl. [dbfz_biomass_heat_capacities](../../digipipe/store/raw/dbfz_biomass_heat_capacities/dataset.md))
+werden in einer Spalte zusammengefasst.
+
+**Dataset: `preprocessed/dbfz_biomass_capacity_rel`**
+
+
+------------------------------
+## Administative areas of Germany
+
+Geodata of administrative areas (Verwaltungsgebiete 1:250 000) extracted,
+reprojected to LAEA Europe(EPSG:3035) and converted to Geopackage.
+
+**Dataset: `preprocessed/bkg_vg250`**
+
diff --git a/docs/datasets/raw_datasets.md b/docs/datasets/raw_datasets.md
new file mode 100644
index 00000000..4288a760
--- /dev/null
+++ b/docs/datasets/raw_datasets.md
@@ -0,0 +1,5030 @@
+# 'Raw' Datasets
+
+------------------------------
+## Technologiedaten
+
+### Jahresvolllaststunden
+
+Anhand typischer heutiger und prognostizierter Werte für Sachsen-Anhalt werden
+folgende Jahresvolllaststunden angenommen:
+
+| Technologie | Jahr | Volllaststunden | Quelle(n) für Annahme | Anmerkung |
+|-----------------|------|----------------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------|
+| Windenergie | 2022 | 1800 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/wind/auswahl/811-durchschnittliche_ja/#goto_811) | |
+| | 2045 | 2300 | [PV- und Windflächenrechner](https://zenodo.org/record/6794558) | |
+| Freiflächen-PV | 2022 | 980 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/solar/auswahl/813-durchschnittliche_ja/#goto_813), [ISE](https://www.ise.fraunhofer.de/content/dam/ise/de/documents/publications/studies/aktuelle-fakten-zur-photovoltaik-in-deutschland.pdf) | |
+| | 2045 | 980 | [PV- und Windflächenrechner](https://zenodo.org/record/6794558), [Ariadne Szenarienreport](https://ariadneprojekt.de/media/2022/02/Ariadne_Szenarienreport_Oktober2021_corr0222_lowres.pdf) | |
+| Aufdach-PV | 2022 | 910 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/solar/auswahl/813-durchschnittliche_ja/#goto_813), [ISE](https://www.ise.fraunhofer.de/content/dam/ise/de/documents/publications/studies/aktuelle-fakten-zur-photovoltaik-in-deutschland.pdf) | |
+| | 2045 | 910 | [Ariadne Szenarienreport](https://ariadneprojekt.de/media/2022/02/Ariadne_Szenarienreport_Oktober2021_corr0222_lowres.pdf) | |
+| Laufwasserkraft | 2022 | 3800 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/wasser/auswahl/840-durchschnittliche_ja/#goto_840) | |
+| | 2045 | 3800 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/wasser/auswahl/840-durchschnittliche_ja/#goto_840) | |
+| Bioenergie | 2022 | 6000 | [foederal-erneuerbar](https://www.foederal-erneuerbar.de/landesinfo/bundesland/ST/kategorie/bioenergie/auswahl/814-durchschnittliche_ja/#goto_814), [ISE](https://www.ise.fraunhofer.de/content/dam/ise/de/documents/publications/studies/DE2018_ISE_Studie_Stromgestehungskosten_Erneuerbare_Energien.pdf) | Bioenergie-Stromerzeugung (ohne
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ image/svg+xml
+
+
+
+
+ Hydrogen Hydrogen Import Hydrogen Export Hydrogen cavern Electrolysis GT Dec. CHP Cen. CCGT Dec. CCGT Cen. CHP
+
+ Biogas upgrading plant Biomass Biomass Import Dec. CHP Cen. CHP Biogas Biogas plant
+
+ Lignite Lignite Import Coal Oven
+
+ Wood Wood Import Wood Oven Cen. CCGT Dec. CCGT
+
+ Methane Import Methane GT Cen. CHP Cen. Gas boiler Dec. Gas boiler Cen. CCGT Dec. CCGT Dec. CHP
+
+ Electricity Import Electricity Export Electricity demand sectors i Electricity Large Battery i = {HH, CTSA, Industry} Wind PV (ground) PV (rooftop) BEV demand Dec. HP Dec. PtH Cen. HP Cen. PtH RoR Central heat Decentral heat Dec. TES Cen. TES Cen. heat demand sectors i Dec. heat demand sectors i Dec. Solar thermal collector Small Battery Cen. Solar thermal collector
+
+
+
diff --git a/docs/img/logos/RLI_DigiPlan_logos.png b/docs/img/logos/RLI_DigiPlan_logos.png
new file mode 100644
index 00000000..1a47339c
Binary files /dev/null and b/docs/img/logos/RLI_DigiPlan_logos.png differ
diff --git a/docs/img/logos/RLI_DigiPlan_logos_horizontal.png b/docs/img/logos/RLI_DigiPlan_logos_horizontal.png
new file mode 100644
index 00000000..d33250d5
Binary files /dev/null and b/docs/img/logos/RLI_DigiPlan_logos_horizontal.png differ
diff --git a/docs/img/digiplan-favicon-512x512.png b/docs/img/logos/digiplan-favicon-512x512.png
similarity index 100%
rename from docs/img/digiplan-favicon-512x512.png
rename to docs/img/logos/digiplan-favicon-512x512.png
diff --git a/docs/img/digiplan-logo-horizontal.png b/docs/img/logos/digiplan-logo-horizontal.png
similarity index 100%
rename from docs/img/digiplan-logo-horizontal.png
rename to docs/img/logos/digiplan-logo-horizontal.png
diff --git a/docs/img/digiplan-logo.png b/docs/img/logos/digiplan-logo.png
similarity index 100%
rename from docs/img/digiplan-logo.png
rename to docs/img/logos/digiplan-logo.png
diff --git a/docs/img/rli_logo.png b/docs/img/logos/rli_logo.png
similarity index 100%
rename from docs/img/rli_logo.png
rename to docs/img/logos/rli_logo.png
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 00000000..526d3fb2
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,6 @@
+mkdocs
+mkdocs-material
+mkdocs-include-markdown-plugin
+mkdocs-same-dir
+mkdocs-exclude
+mkdocs-exclude-search
\ No newline at end of file
diff --git a/docs/sections/background.md b/docs/sections/background.md
new file mode 100644
index 00000000..6397b0d4
--- /dev/null
+++ b/docs/sections/background.md
@@ -0,0 +1,35 @@
+# Hintergrund
+
+Die Energiewende kann nur gemeinsam erreicht werden. Wie aber können alle
+Interessensgruppen ihre Sichtweisen und Bedürfnisse in die Energiewendeplanung
+einbringen?
+Das [Reiner Lemoine Institut (RLI)](https://reiner-lemoine-institut.de) hat
+dieses
+Stakeholder-Empowerment-Tool mit Unterstützung
+der [Energieavantgarde Anhalt e.V.](https://www.energieavantgarde.de/)
+entwickelt, das es Akteurinnen und Akteuren der Energiewende
+ermöglicht, sich an Planungsprozessen zu beteiligen.
+
+Das Ziel ist die Beförderung des Klimaschutzes in ländlichen Regionen durch eine
+effiziente und transparente Steuerung. Hierzu sollen dezentrale Energiesysteme
+gestärkt werden, um beispielsweise hohe Netzentgelte für ländliche Regionen zu
+vermeiden und die Akzeptanz des EE-Angebotes zu fördern. In einem dezentralen
+Energiesystem können unfaire Kosten vermieden und die BürgerInnen zum Prosuming
+ermutigt werden. Dies erfordert sorgfältiges Planen – auch über aktuelle
+Markttrends hinaus. Dadurch entsteht eine höhere Akzeptanz durch eigenes und
+gemeinsames Handeln und Verhandeln, wovon der Klimaschutz profitiert.
+
+Das Tool kann zur Unterstützung regionaler Planungsorganisationen und Landkreise
+sowie für die Partizipation der Zivilgesellschaft eingesetzt werden, um die
+Implementierung von Erneuerbaren Energien effizienter zu planen und den Zubau
+besser und schneller zu steuern. Es ermöglicht BürgerInnen und Administrationen
+den Zugriff auf den Energiekonsum und die genutzten und verfügbaren
+Energiequellen in ihrer jeweiligen Region. Dies gewährleistet ein besseres
+Verständnis der aktuellen Lage und hilft, Bedarfe und Potenziale zu erkunden
+sowie Ausbaupfade zu verhandeln.
+
+## Förderung
+
+Das Projekt wurde im Rahmen des Projekts
+[Heimat 2.0](https://www.bbsr.bund.de/BBSR/DE/forschung/programme/region-gestalten/initiativen/2020/heimat-2-0/01-start.html)
+im Programm [Region gestalten](http://www.regiongestalten.de/) gefördert.
diff --git a/docs/sections/data.md b/docs/sections/data.md
new file mode 100644
index 00000000..89c91d51
--- /dev/null
+++ b/docs/sections/data.md
@@ -0,0 +1,7 @@
+{%
+ include-markdown "../../digipipe/store/DATASETS.md"
+%}
+
+## Datasets
+
+See [Datasets](../datasets/raw_datasets.md)
diff --git a/docs/sections/data_overview.md b/docs/sections/data_overview.md
new file mode 100644
index 00000000..c4e3951b
--- /dev/null
+++ b/docs/sections/data_overview.md
@@ -0,0 +1,18 @@
+# Überblick
+
+Hier werden die zugrunde liegende und abgeleiteten Daten sowie die Methoden
+beschrieben.
+
+Grundlegender Datenfluss:
+
+0. [Rohdaten (RAW)](../datasets/raw_datasets.md) -->
+1. [Vorverarbeitete Daten (PREPROCESSED)](../datasets/preprocessed_datasets.md)
+ -->
+2. [Fertige Datensätze (DATASETS)](../datasets/datasets_datasets.md) -->
+3. [App-fertige Datensätze (APPDATA)](../datasets/appdata_datasets.md)
+
+Für eine detaillierte Beschreibung des Datenflusses siehe
+[Data Pipeline](data.md).
+
+Die Datensätze und dazugehörigen Methoden sind in den Kategorien im links
+dargestellte Menü abrufbar.
diff --git a/docs/sections/docs.md b/docs/sections/docs.md
new file mode 100644
index 00000000..13745987
--- /dev/null
+++ b/docs/sections/docs.md
@@ -0,0 +1,8 @@
+# How to build the docs
+
+With activated env:
+
+```
+python docs/generate_dataset_mds.py
+mkdocs build
+```
diff --git a/docs/sections/esys.md b/docs/sections/esys.md
new file mode 100644
index 00000000..5a714aaa
--- /dev/null
+++ b/docs/sections/esys.md
@@ -0,0 +1,192 @@
+# Energiesystem
+
+Im Folgenden wird der Aufbau des in Digiplan verwendeten Energiesystems für die
+Region Anhalt-Bitterfeld-Wittenberg beschrieben. Die Generierung der
+Energiesystemdaten für die App ist Teil der
+[Developer Docs](../sections/esys_dev.md).
+
+## Das Energiesystem im Überblick
+
+Schematische Darstellung:
+
+
+
+In den Modellen wird das Energiesystem für das Jahr 2045 aufgestellt. Dabei
+wird sich am derzeitigen Energiesystem orientiert, dessen Komponenten u.a. auf
+den Daten des Marktstammdatenregisters (MaStR) beruhen.
+
+## Annahmen und Vereinfachungen
+
+Es werden die folgenden Annahmen und Vereinfachungen getroffen:
+
+- Für die Prognosen (z.B. Energieträger, Verbräuche) werden die
+ [BMWK Langfristszenarien](https://langfristszenarien.de) verwendet, vgl.
+ Datensatz [BMWK Langfristszenarien](../datasets/raw_datasets.md#bmwk-langfristszenarien).
+- Biomasse wird unterteilt in: feste, gasförmige und flüssige Biomasse.
+- Die Vorhersage zur Entwicklung der Biomasse geschieht auf der Basis von:
+ ["Technoökonomische Analyse und Transformationspfade des energetischen Biomassepotentials (TATBIO)"](https://www.bmwk.de/Redaktion/DE/Publikationen/Studien/technooekonomische-analyse-und-transformationspfade-des-energetischen-biomassepotentials.pdf?__blob=publicationFile&v=1),
+ vgl. Datensatz [dbfz_biomass_heat_capacities](../datasets/raw_datasets.md#installierte-leistungen-von-biomasse-konversionstechnologien).
+- Flüssige Biomasse wird im Jahr 2045 ausschließlich im Verkehrsektor
+ eingesetzt.
+- Der Verkehrssektor wird vernachlässigt. Demnach entfällt der Einsatz flüssiger
+ Biomasse.
+- Es wird kein Biogas importiert: Sämtliches Biogas wird regional durch Biomasse
+ erzeugt.
+- Der Bus Methan steht stellvertretend für einen Bus für klimaneutrale Gase.
+ Es handelt sich um eine Mischung aus grünem Wasserstoff und Biomethan.
+ Immer wenn von Methan gesprochen wird, ist diese Gasmischung gemeint.
+- Der Energieträger Braunkohle ist nur relevant zur Berechnungen der Emissionen
+ zum Status quo.
+- Für das Jahr 2045 ist kein Import von Braunkohle vorhergesehen.
+- Es wird das Jahr 2045 in stündlichen Schritten betrachtet.
+- Sämtliche Zeitreihen liegen in dieser Auslösung vor oder werden in diese
+ Auflösung übersetzt.
+- Es handelt sich um eine Dispatch Optimierung.
+- Sämtliche Kapazitäten und Energiemengen bis auf die der Importe und Exporte
+ sowie die der Speichertechnologien sind fix gesetzt.
+- Die Optimierung findet nach dem Perfect-Forsight-Prinzip über den
+ Zeithorizont eines Jahres statt.
+- Die Strombedarfe werden je Sektor (Haushalte, GHD, Industrie) aggregiert
+ abgebildet, vgl. Datensatz
+ [demand_electricity_region](../datasets/datasets_datasets.md#strombedarf).
+- Die Wärmebedarfe werden je Sektor (Haushalte, GHD, Industrie) abgebildet,
+ unterteilt in zentral (Fernwärmenetze) und dezentral, vgl. Datensatz
+ [demand_heat_region](../datasets/datasets_datasets.md#wärmebedarf).
+- Es erfolgt keine Modellierung der Strom- und Wärmenetze.
+
+## Das Energiesystem im Detail
+
+### Busse
+
+Die folgenden acht Energieträger sind die Busse im Energiesystem:
+
+- Elektrizität
+- Zentrale Wärme
+- Dezentrale Wärme
+- Methan
+- Holz
+- Braunkohle
+- Biomassse
+- Biogas
+
+### Erzeugungstechnologien
+
+Volatile Erzeugungsanlagen werden mit der
+[oemof.tabular Komponente volatile](https://github.com/oemof/oemof-tabular/blob/dev/src/oemof/tabular/facades/volatile.py)
+abgebildet. Die folgenden Erzeugungsanlagen werden im Energiesystem betrachtet:
+
+| Komponente | Energieträger input | Energieträger output | Beschreibung | Komponentenname
+|------------------------------|---------------------|----------------------|-------------------------|----------------------------------
+| Wind | | Elektrizität | Onshore-Wind | wind-onshore
+| PV (ground) | | Elektrizität | Freiflächen-PV | solar-pv_ground
+| PV (rooftop) | | Elektrizität | Aufdach-PV | solar-pv_rooftop
+| Cen. Solar thermal collector | | Elektrizität | Zentrale Solarthermie | solar-thermalcollector_central
+| Dec. Solar thermal collector | | Elektrizität | Dezentrale Solarthermie | solar-thermalcollector_decentral
+| RoR | | Elektrizität | Laufwasserkraftwerk | hydro-ror
+
+
+### Import und Export
+
+Strombezug aus dem Stromnetz wird mittels einer
+[oemof.tabular Komponente shortage](https://github.com/oemof/oemof-tabular/blob/dev/src/oemof/tabular/facades/shortage.py)
+abgebildet.
+Mit dieser Komponente wird auch der Bezug von Biomethan abgebildet und der von
+Holz und etwaiger Braunkohle.
+
+| Komponente | Energieträger input | Energieträger output | Beschreibung | Komponentenname
+|--------------------|---------------------|----------------------|-------------------------|--------------------
+| Electricity Import | | Elektrizität | Bezug aus dem Stromnetz | electricity-import
+| Methane Import | | Methan | Bezug von Methan | ch4-import
+| Wood Import | | Holz | Bezug von Holz | wood-import
+| Lignite Import | | Braunkohle | Bezug von Braunkohle | lignite-import
+| Biomass Import | | Biomasse | Bezug von Biomasse | biomass-import
+
+Es wird der Export von Strom berücksichtigt. Der Verkauf von Strom aus der
+Region wird mit der
+[oemof.tabular Komponente excess](https://github.com/oemof/oemof-tabular/blob/dev/src/oemof/tabular/facades/excess.py)
+abgebildet.
+
+| Komponente | Energieträger input | Energieträger output | Beschreibung | Komponentenname
+|--------------------|---------------------|----------------------|-------------------|--------------------
+| Electricity Export | Elektrizität | | Verkauf von Strom | electricity-export
+
+### Bedarf
+
+Die Sektoren Haushalte (HH), Gewerbe, Handel und Dienstleistungen (CTS) sowie
+Industrie (Industry) haben einen Bedarf an Elektrizität, zentraler Wärme und
+dezentraler Wärme. Der jeweilige Bedarf wird im Modell mit der
+[oemof.tabular Komponente load](https://github.com/oemof/oemof-tabular/blob/dev/src/oemof/tabular/facades/load.py)
+abgebildet.
+
+| Komponente | Energieträger input | Energieträger output | Beschreibung | Komponentenname
+|------------------------------------|---------------------|----------------------|---------------------------------------|---------------------------
+| BEV demand | Elektrizität | | Strombedarf für Elektrofahrzeuge | electricity-bev_charging
+| Electricity demand sector HH | Elektrizität | | Strombedarf Haushalte | electricity-demand_hh
+| Electricity demand sector CTS | Elektrizität | | Strombedarf CTS | electricity-demand_cts
+| Electricity demand sector Industry | Elektrizität | | Strombedarf Industrie | electricity-demand_ind
+| Cen. heat demand sector HH | Zentrale Wärme | | Bedarf an zentraler Wärme Haushalte | heat_central-demand_hh
+| Cen. heat demand sector CTS | Zentrale Wärme | | Bedarf an zentraler Wärme CTS | heat_central-demand_cts
+| Cen. heat demand sector Industry | Zentrale Wärme | | Bedarf an zentraler Wärme Industrie | heat_central-demand_ind
+| Dec. heat demand sector HH | Dezentrale Wärme | | Bedarf an dezentraler Wärme Haushalte | heat_decentral-demand_hh
+| Dec. heat demand sector CTS | Dezentrale Wärme | | Bedarf an dezentraler Wärme CTS | heat_decentral-demand_cts
+| Dec. heat demand sector Industry | Dezentrale Wärme | | Bedarf an dezentraler Wärme Industrie | heat_decentral-demand_ind
+
+### Wärmeerzeuger und weitere Wandler
+
+Wärmeerzeuger und weitere Wandler werden mittels der
+[oemof.tabular Komponente conversion](https://github.com/oemof/oemof-tabular/blob/dev/src/oemof/tabular/facades/conversion.py)
+eingebunden.
+
+| Komponente | Energieträger input | Energieträger output | Beschreibung | Komponentenname
+|------------------------|---------------------|----------------------|----------------------------|--------------------------------
+| GT | Methan | Elektrizität | Gasturbine | ch4-gt
+| Cen. HP | Elektrizität | Zentrale Wärme | Zentrale Wärmepumpe | electricity-heatpump_central
+| Dec. HP | Elektrizität | Dezentrale Wärme | Dezentrale Wärmepumpe | electricity-heatpump_decentral
+| Cen. PtH | Elektrizität | Zentrale Wärme | Zentrale Power-to-Heat | electricity-pth_central
+| Dec. PtH | Elektrizität | Dezentrale Wärme | Dezentrale Power-to-Heat | electricity-pth_decentral
+| Cen. Gas boiler | Methan | Zentrale Wärme | Zentrale Gasboiler | ch4-boiler_central
+| Dec. Gas boiler | Methan | Dezentrale Wärme | Dezentrale Gasboiler | ch4-boiler_decentral
+| Wood Oven | Holz | Dezentrale Wärme | Dezentraler Holzofen | wood-oven
+| Coal Oven | Braunkohle | Dezentrale Wärme | Dezentraler Braunkohleofen | lignite-oven
+| Biogas plant | Biomasse | Biogas | Biogasanlage | biomass-biogas_plant
+| Biogas upgrading plant | Biogas | Methan | Biogasaufbereitung | biogas-biogas_upgrading_plant
+
+### Kraft-Wärme-Kopplung
+
+Bei der Kraft-Wärme-Kopplung wird zwischen Gas-Turbinen mit Abhitzekessel und
+Blockheizkraftwerken unterschieden. Für Gas-Turbinen mit Abhitzekessel wird die
+[oemof.tabular Komponente Komponente extraction_turbine](https://github.com/oemof/oemof-tabular/blob/dev/src/oemof/tabular/facades/extraction_turbine.py)
+verwendet.
+
+| Komponente | Energieträger input | Energieträger output | Beschreibung | Komponentenname
+|------------|---------------------|-----------------------------------|------------------------------------------|-----------------------
+| Cen. CCGT | Methan | Elektrizität und Zentrale Wärme | Zentrale Gasturbinen mit Abhitzekessel | ch4-extchp_central
+| Dec. CCGT | Methan | Elektrizität und Dezentrale Wärme | Dezentrale Gasturbinen mit Abhitzekessel | ch4-extchp_decentral
+| Cen. CCGT | Holz | Elektrizität und Zentrale Wärme | Zentrale Gasturbinen mit Abhitzekessel | wood-extchp_central
+| Dec. CCGT | Holz | Elektrizität und Dezentrale Wärme | Dezentrale Gasturbinen mit Abhitzekessel | wood-extchp_decentral
+
+
+Für Blockheizkraftwerke wir auf die
+[oemof.tabular Komponente Komponente backpressure_turbine](https://github.com/oemof/oemof-tabular/blob/dev/src/oemof/tabular/facades/backpressure_turbine.py)
+zurückgegriffen, welche einer Conversion Komponente mit zwei Ausgängen ähnelt
+und daher vereinfachend zum Abbilden eines BHKWs verwendet werden kann.
+
+| Komponente | Energieträger input | Energieträger output | Beschreibung | Komponentenname
+|------------|---------------------|-----------------------------------|------------------------------------------|----------------------------------
+| Cen. CHP | Methan | Elektrizität und Zentrale Wärme | Zentrales Blockheizkraftwerk | ch4-bpchp_central
+| Dec. CHP | Methan | Elektrizität und Dezentrale Wärme | Dezentrales Blockheizkraftwerk | ch4-bpchp_decentral
+| Cen. CHP | Biogas | Elektrizität und Zentrale Wärme | Zentrales Blockheizkraftwerk | biogas-bpchp_central
+| Dec. CHP | Biogas | Elektrizität und Dezentrale Wärme | Dezentrales Blockheizkraftwerk | biogas-bpchp_decentral
+
+### Speicher
+
+Die folgenden Speicher sind im Energiesystem von ABW mittels der
+[oemof.tabular Komponente Komponente storage](https://github.com/oemof/oemof-tabular/blob/dev/src/oemof/tabular/facades/storage.py)
+eingebunden:
+
+| Komponente | Energieträger input | Energieträger output | Beschreibung | Komponentenname
+|---------------|---------------------|----------------------|---------------------------------------|---------------------------------
+| Small Battery | Elektrizität | Elektrizität | Heimbatterien | electricity-small_scale_battery
+| Large Battery | Elektrizität | Elektrizität | Großbatterien | electricity-large_scale_battery
+| Cen. TES | Zentrale Wärme | Zentrale Wärme | Zentrale thermische Energiespeicher | heat_central-storage
+| Dec. TES | Dezentrale Wärme | Dezentrale Wärme | Dezentrale thermische Energiespeicher | heat_decentral-storage
diff --git a/docs/sections/esys_dev.md b/docs/sections/esys_dev.md
new file mode 100644
index 00000000..6a81262f
--- /dev/null
+++ b/docs/sections/esys_dev.md
@@ -0,0 +1,3 @@
+{%
+ include-markdown "../../digipipe/esys/ESYS.md"
+%}
\ No newline at end of file
diff --git a/docs/sections/installation.md b/docs/sections/installation.md
new file mode 100644
index 00000000..5c503d58
--- /dev/null
+++ b/docs/sections/installation.md
@@ -0,0 +1,61 @@
+# Installation
+
+**Note: Linux only, Windows is currently not supported.**
+
+First, clone via SSH using
+
+ git clone git@github.com:rl-institut-private/digipipe.git /local/path/to/digipipe/
+
+Prerequisite:
+
+- _conda_ ([Anaconda](https://docs.anaconda.com/anaconda/install/)
+/[Miniconda](https://conda.io/en/latest/miniconda.html))
+- `python ",
+ "Marie-Claire Gering ",
+ "David Starzl ",
+ "Aaron Schilling ",
+ "Hedwig Bartels ",
+]
+license = 'GNU AGPLv3'
+classifiers = [ # Optional
+ # How mature is this project? Common values are
+ # 3 - Alpha
+ # 4 - Beta
+ # 5 - Production/Stable
+ "Development Status :: 5 - Production/Stable",
+ # Indicate who your project is intended for
+ "Intended Audience :: Developers",
+ "Topic :: Software Development :: Build Tools",
+ # Pick your license as you wish
+ "License :: OSI Approved :: AGPL License",
+ # Specify the Python versions you support here. In particular, ensure
+ # that you indicate whether you support Python 2, Python 3 or both.
+ # These classifiers are *not* checked by 'pip install'. See instead
+ # 'python_requires' below.
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.6",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+]
+readme = "README.md"
+repository = "https://github.com/rl-institut/digipipe"
+
+[tool.poetry.urls]
+"Bug Tracker" = "https://github.com/rl-institut/digipipe/issues"
+
+[tool.poetry.dependencies]
+python = ">=3.8,<3.12"
+pandas = "^1.5.3"
+geopandas = "^0.12.2"
+shapely = "^2.0.1"
+snakemake = "7.32.0"
+matplotlib = "^3.7.0"
+pyyaml = "^6.0"
+descartes = "^1.1.0"
+fiona = "<=1.9"
+rtree = "^1.0.1"
+openpyxl = "3.1.0"
+plotly = "^5.13.1"
+rasterstats = "^0.18.0"
+networkx = "^3.0"
+pygraphviz = "^1.10"
+pygments = "^2.14.0"
+rasterio = "^1.3.6"
+jinja2 = "3.0.3"
+geopy = "^2.3.0"
+oemoflex = { git = "https://git@github.com/rl-institut/oemoflex", tag = "0.0.1"}
+ruff = "^0.0.257"
+types-pyyaml = "^6.0.12.9"
+pyxlsb = "^1.0.10"
+tabula-py = "^2.7.0"
+requests = "^2.30.0"
+types-requests = "^2.30.0.0"
+mkdocs = "^1.4.3"
+mkdocs-material = "^9.1.12"
+mkdocs-include-markdown-plugin = "^4.0.4"
+mkdocs-same-dir = "^0.1.2"
+mkdocs-exclude = "^1.0.2"
+mkdocs-exclude-search = "^0.6.5"
+boto3 = "1.26.125"
+
+[tool.poetry.dev-dependencies]
+m2r2 = "^0.3.3"
+
+
+# Testing
+# ------------------------------------------------------------------------------
+# mypy = "^0.812" # https://github.com/python/mypy
+pytest = "^6.2.4" # https://github.com/pytest-dev/pytest
+
+
+# Code quality
+# ------------------------------------------------------------------------------
+flake8 = "^3.9.2" # https://github.com/PyCQA/flake8
+pylint-django = "^2.3.0" # https://github.com/PyCQA/pylint-django # downgraded because https://github.com/PyCQA/pylint-django/issues/309
+pre-commit = "^2.13.0" # https://github.com/pre-commit/pre-commit
+flake8-bandit = "^3.0.0"
+flake8-bugbear = "^22.4.25"
+flake8-builtins = "^1.5.3"
+flake8-comprehensions = "^3.10.0"
+darglint = "^1.8.1"
+flake8-eradicate = "^1.2.1"
+flake8-isort = "^4.1.1"
+flake8-pytest-style = "^1.6.0"
+black = "^21.12b0"
+mypy = "^1.0.1"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.black]
+line-length = 80
+
+[tool.isort]
+profile = "black"
+line_length = 80
+
+[tool.ruff]
+line-length = 80
+ignore = [
+ "F821", # Undefined name
+]
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 9a53b4b7..00000000
--- a/requirements.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-pandas
-geopandas
-shapely
-snakemake
-matplotlib
-pyyaml
-descartes
-fiona
-rtree
-plotly
-rasterstats
-networkx
-pygraphviz
-pygments
-rasterio
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 70727073..00000000
--- a/setup.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from setuptools import setup, find_packages
-from os import path
-
-here = path.abspath(path.dirname(__file__))
-
-# Get the long description from the README file
-with open(path.join(here, "README.md"), encoding="utf-8") as f:
- long_description = f.read()
-
-with open('requirements.txt') as f:
- requirements = f.read().splitlines()
-
-setup(
- name="digipipe",
- version="0.0.0",
- description="Pipeline for data and energy system in the Digiplan project.",
- long_description=long_description,
- long_description_content_type="text/markdown",
- url="https://github.com/rl-institut/digipipe/",
- author="Reiner Lemoine Institut",
- author_email='jonathan.amme@rl-institut.de',
- license='GNU AGPLv3',
- classifiers=[ # Optional
- # How mature is this project? Common values are
- # 3 - Alpha
- # 4 - Beta
- # 5 - Production/Stable
- "Development Status :: 3 - Alpha",
- # Indicate who your project is intended for
- "Intended Audience :: Developers",
- "Topic :: Software Development :: Build Tools",
- # Pick your license as you wish
- "License :: OSI Approved :: AGPL License",
- # Specify the Python versions you support here. In particular, ensure
- # that you indicate whether you support Python 2, Python 3 or both.
- # These classifiers are *not* checked by 'pip install'. See instead
- # 'python_requires' below.
- "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.6",
- "Programming Language :: Python :: 3.7",
- "Programming Language :: Python :: 3.8",
- ],
- packages=find_packages(),
- python_requires=">=3, <4",
- install_requires=requirements,
- #extras_require={"dev": [], "test": []}, # Optional
- project_urls={ # Optional
- "Bug Reports": "https://github.com/rl-institut/digipipe/issues",
- "Source": "https://github.com/rl-institut/digipipe",
- },
-)
+ 
+
-
+