first test of django app

.gitignore

@@ -166,3 +166,4 @@ doc/_build
 .vscode/settings.json
 data/silver/*.nc
 data/silver/weather_forecasts/*.nc
+.vscode/settings.json

pyproject.toml

@@ -74,6 +74,15 @@ extra-dependencies = [
   "streamlit-analytics2",
 ]
 
+# new env for django
+[tool.hatch.envs.django]
+extra-dependencies = [
+  "django",
+  "psycopg2-binary",
+  "django-crontab",
+  "django-environ",
+  "django-debug-toolbar",
+]
 [tool.hatch.envs.serve.scripts]
 prod = "streamlit run --server.port 8502 --browser.gatherUsageStats false --server.runOnSave true --server.baseUrlPath energy_forecast  src/energy_forecast/dashboard/Accueil.py"
 
@@ -98,3 +107,4 @@ exclude_lines = [
 
 [tool.ruff]
 extend-include = ["*.ipynb"]
+

src/webapp/core/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

src/webapp/core/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class CoreConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'core'

src/webapp/core/cron.py

@@ -0,0 +1,8 @@
+# core/cron.py
+
+from .models import Forecast
+from datetime import date, timedelta
+
+def update_forecast_data():
+    # Fetch or generate forecast data for the next 3 days
+    # pass

src/webapp/core/data_management/etl_weather.py

@@ -0,0 +1,26 @@
+"""This module contains the fonctionalities to extract, transform and load weather data."""
+
+import pandas as pd
+from pathlib import Path
+from core.models import WeatherForecastMeanDepartment
+from energy_forecast.meteo import ArpegeSimpleAPI
+
+def run_etl_pipeline():
+    """Run the ETL pipeline to load weather data into the database."""
+
+    # extract
+    arpege_client = ArpegeSimpleAPI()
+    departements_sun_flux = arpege_client.departement_sun()
+    departements_wind_speed = arpege_client.departement_wind()
+
+    all_data = departements_sun_flux.join(departements_wind_speed, how='inner').reset_index()
+    print(all_data)
+    for _, row in all_data.iterrows():
+        WeatherForecastMeanDepartment.objects.create(
+            valid_time=row['valid_time'],
+            department=row['departement'],
+            horizon_forecast='D1',
+            sun_flux=row['sun_flux'],
+            temperature=row['temperature'],
+            wind_speed=row['wind_speed']
+        )

src/webapp/core/management/commands/load_historical_data.py

@@ -0,0 +1,34 @@
+from django.core.management.base import BaseCommand
+from core.models import WeatherForecastMeanDepartment
+from energy_forecast import ROOT_DIR
+import pandas as pd
+
+class Command(BaseCommand):
+    help = 'Load weather forecasts from a CSV file into the database'
+
+    def handle(self, *args, **kwargs):
+        data_root = ROOT_DIR / 'data' / 'silver' / 'weather_forecasts'
+        sun_flux_file = data_root / 'sun_flux_downward_hourly_d1_departements.csv'
+        temperature_file = data_root / 'temperature_hourly_d1_departements.csv'
+        wind_speed_file = data_root / 'wind_speed_hourly_d1_departements.csv'
+
+        sun_flux_df = pd.read_csv(sun_flux_file)[['valid_time', 'departement', 'sun_flux']].set_index(['valid_time', 'departement'])
+        # temperature_df = pd.read_csv(temperature_file)[['valid_time', 'departement', 'temperature']].set_index(['valid_time', 'departement'])
+        wind_speed_df = pd.read_csv(wind_speed_file)[['valid_time', 'departement', 'wind_speed']].set_index(['valid_time', 'departement'])
+        print(sun_flux_df)
+        # print(temperature_df)
+        print(wind_speed_df)
+        all_data = sun_flux_df.join(wind_speed_df, how='inner').reset_index()
+        print(all_data)
+        for _, row in all_data.iterrows():
+            WeatherForecastMeanDepartment.objects.create(
+                valid_time=row['valid_time'],
+                department=row['departement'],
+                horizon_forecast='D1',
+                sun_flux=row['sun_flux'],
+                temperature=0,
+                wind_speed=row['wind_speed']
+            )
+
+
+        self.stdout.write(self.style.SUCCESS('Successfully loaded weather forecasts'))

src/webapp/core/migrations/0001_initial.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.1.1 on 2024-09-06 12:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Forecast',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('date', models.DateField()),
+                ('forecast_data', models.TextField()),
+            ],
+        ),
+    ]

src/webapp/core/migrations/0002_weatherforecastmeandepartment.py

@@ -0,0 +1,28 @@
+# Generated by Django 5.1.1 on 2024-09-07 05:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='WeatherForecastMeanDepartment',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('valid_time', models.DateTimeField()),
+                ('department', models.CharField(choices=[('Aisne ', 'Aisne '), ('Aube ', 'Aube '), ('Calvados ', 'Calvados '), ('Cantal ', 'Cantal '), ('Eure-et-Loir ', 'Eure-et-Loir '), ('Ille-et-Vilaine ', 'Ille-et-Vilaine '), ('Jura ', 'Jura '), ('Landes ', 'Landes '), ('Loire ', 'Loire '), ('Loiret ', 'Loiret '), ('Lot-et-Garonne ', 'Lot-et-Garonne '), ('Meuse ', 'Meuse '), ('Orne ', 'Orne '), ('Pas-de-Calais ', 'Pas-de-Calais '), ('Puy-de-Dôme ', 'Puy-de-Dôme '), ('Bas-Rhin ', 'Bas-Rhin '), ('Haut-Rhin ', 'Haut-Rhin '), ('Seine-Maritime ', 'Seine-Maritime '), ('Yonne ', 'Yonne '), ('Seine-Saint-Denis ', 'Seine-Saint-Denis '), ('Alpes-de-Haute-Provence ', 'Alpes-de-Haute-Provence '), ('Hautes-Alpes ', 'Hautes-Alpes '), ('Ardèche ', 'Ardèche '), ('Ardennes ', 'Ardennes '), ('Ariège ', 'Ariège '), ('Charente-Maritime ', 'Charente-Maritime '), ('Corrèze ', 'Corrèze '), ('Dordogne ', 'Dordogne '), ('Eure ', 'Eure '), ('Indre-et-Loire ', 'Indre-et-Loire '), ('Lozère ', 'Lozère '), ('Nièvre ', 'Nièvre '), ('Oise ', 'Oise '), ('Pyrénées-Atlantiques ', 'Pyrénées-Atlantiques '), ('Rhône ', 'Rhône '), ('Saône-et-Loire ', 'Saône-et-Loire '), ('Paris ', 'Paris '), ('Yvelines ', 'Yvelines '), ('Tarn ', 'Tarn '), ('Tarn-et-Garonne ', 'Tarn-et-Garonne '), ('Var ', 'Var '), ('Vendée ', 'Vendée '), ('Haute-Vienne ', 'Haute-Vienne '), ('Vosges ', 'Vosges '), ('Hauts-de-Seine ', 'Hauts-de-Seine '), ('Allier ', 'Allier '), ('Alpes-Maritimes ', 'Alpes-Maritimes '), ('Aude ', 'Aude '), ('Corse-du-Sud ', 'Corse-du-Sud '), ("Côtes-d'Armor ", "Côtes-d'Armor "), ('Creuse ', 'Creuse '), ('Doubs ', 'Doubs '), ('Finistère ', 'Finistère '), ('Gard ', 'Gard '), ('Gironde ', 'Gironde '), ('Indre ', 'Indre '), ('Isère ', 'Isère '), ('Marne ', 'Marne '), ('Haute-Marne ', 'Haute-Marne '), ('Moselle ', 'Moselle '), ('Hautes-Pyrénées ', 'Hautes-Pyrénées '), ('Pyrénées-Orientales ', 'Pyrénées-Orientales '), ('Savoie ', 'Savoie '), ('Haute-Savoie ', 'Haute-Savoie '), ('Seine-et-Marne ', 'Seine-et-Marne '), ('Vaucluse ', 'Vaucluse '), ('Vienne ', 'Vienne '), ('Val-de-Marne ', 'Val-de-Marne '), ('Ain ', 'Ain '), ('Aveyron ', 'Aveyron '), ('Bouches-du-Rhône ', 'Bouches-du-Rhône '), ('Charente ', 'Charente '), ('Cher ', 'Cher '), ('Haute-Corse ', 'Haute-Corse '), ("Côte-d'Or ", "Côte-d'Or "), ('Drôme ', 'Drôme '), ('Haute-Garonne ', 'Haute-Garonne '), ('Gers ', 'Gers '), ('Hérault ', 'Hérault '), ('Haute-Loire ', 'Haute-Loire '), ('Loire-Atlantique ', 'Loire-Atlantique '), ('Lot ', 'Lot '), ('Maine-et-Loire ', 'Maine-et-Loire '), ('Manche ', 'Manche '), ('Morbihan ', 'Morbihan '), ('Nord ', 'Nord '), ('Haute-Saône ', 'Haute-Saône '), ('Sarthe ', 'Sarthe '), ('Somme ', 'Somme '), ('Essonne ', 'Essonne '), ("Val-d'Oise ", "Val-d'Oise "), ('Loir-et-Cher ', 'Loir-et-Cher '), ('Mayenne ', 'Mayenne '), ('Meurthe-et-Moselle ', 'Meurthe-et-Moselle ')], max_length=50)),
+                ('horizon_forecast', models.CharField(choices=[('D0', 'D0'), ('D1', 'D1'), ('D2', 'D2'), ('D3', 'D3')], max_length=2)),
+                ('sun_flux', models.FloatField()),
+                ('wind_speed', models.FloatField()),
+                ('temperature', models.FloatField()),
+            ],
+            options={
+                'indexes': [models.Index(fields=['valid_time', 'horizon_forecast'], name='core_weathe_valid_t_82976e_idx')],
+            },
+        ),
+    ]

src/webapp/core/models.py

@@ -0,0 +1,38 @@
+from django.db import models
+from energy_forecast.constants import departements
+
+HORIZON_CHOICES = [
+    ('D0', 'D0'),
+    ('D1', 'D1'),
+    ('D2', 'D2'),
+    ('D3', 'D3')
+]
+
+# Create your models here.
+class WeatherForecastMeanDepartment(models.Model):
+    # set valide_time as datetime 
+    # set sun_flux as a float
+    # set department as a category
+    # set the horizon forcast type : D0, D1, D2, D3
+    # use datetime and department as a index
+    valid_time = models.DateTimeField()
+    department = models.CharField(max_length=50, choices=[(d, d) for d in departements])
+    horizon_forecast = models.CharField(max_length=2, choices=HORIZON_CHOICES)
+    sun_flux = models.FloatField()
+    wind_speed = models.FloatField()
+    temperature = models.FloatField()
+
+    class Meta:
+        indexes = [
+            models.Index(fields=['valid_time', 'horizon_forecast']),
+        ]
+
+    def __str__(self):
+        return f"{self.valid_time} - {self.horizon_forecast} - {self.department} - {self.sun_flux} "
+
+class Forecast(models.Model):
+    date = models.DateField()
+    forecast_data = models.TextField()
+
+    def __str__(self):
+        return f"{self.date}: {self.forecast_data}"

src/webapp/core/templates/landing_page.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Energy Forecast</title>
+</head>
+<body>
+    <h1>Energy Forecast for the Next 3 Days</h1>
+    <ul>
+        {% for forecast in forecasts %}
+            <li>{{ forecast.date }}: {{ forecast.forecast_data }}</li>
+        {% endfor %}
+    </ul>
+</body>
+</html>

src/webapp/core/templates/registration/login.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Login</title>
+</head>
+<body>
+    <h2>Login</h2>
+    <form method="post">
+        {% csrf_token %}
+        {{ form.as_p }}
+        <button type="submit">Login</button>
+    </form>
+</body>
+</html>

src/webapp/core/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

src/webapp/core/views.py

@@ -0,0 +1,28 @@
+from django.shortcuts import render
+from .models import WeatherForecastMeanDepartment
+from datetime import date, timedelta
+from django.http import HttpResponse
+
+from .data_management.etl_weather import run_etl_pipeline
+
+def landing_page(request):
+    pass
+    #return render(request, 'landing_page.html', {'forecasts': forecasts})
+
+
+
+def check_and_load_data(request):
+    # Check if data exists
+    today = date.today()
+    # check if data exists for today
+    if WeatherForecastMeanDepartment.objects.filter(valid_time__date=today).exists():
+        return HttpResponse("Data already exists for today.")
+    else :
+        # Run ETL pipeline if data does not exist
+        run_etl_pipeline()
+
+    # Load data from the database
+    data = WeatherForecastMeanDepartment.objects.all()
+
+    # Render the data in a template or return as a response
+    return HttpResponse(f"Loaded {data.count()} records from the database.")

src/webapp/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'webapp.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

src/webapp/webapp/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for webapp project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'webapp.settings')
+
+application = get_asgi_application()