From e94fc369067274fd88abf2c0e5dc244e227bcffc Mon Sep 17 00:00:00 2001 From: gaudinnicolas Date: Wed, 20 Nov 2024 14:20:31 +0100 Subject: [PATCH] fix: missing code written since august --- .d4g-tools/bin/.from-scratch-sh | 65 +++++++++ .d4g-tools/bin/d4g.env | 1 + .d4g-tools/bin/docker.sh | 69 ++++++++++ .d4g-tools/bin/env.sh | 54 ++++++++ .d4g-tools/bin/fastapi/main.py | 29 ++++ .d4g-tools/bin/fastapi/readme.md | 160 ++++++++++++++++++++++ .d4g-tools/bin/python/pyproject.toml.dist | 2 +- .d4g-tools/bin/taipy/DockerFile | 20 +++ .d4g-tools/bin/taipy/compose.yml | 19 +++ .d4g-tools/bin/taipy/requirements.txt | 0 .d4g-tools/bin/uwsgi/uwsgi.sh | 3 +- .d4g-tools/lib/depends.sh | 1 + .d4g-tools/lib/model/country.py | 16 +++ .github/workflows/pre-commit copy.yaml | 25 ++++ d4g.ini | 7 +- 15 files changed, 465 insertions(+), 6 deletions(-) create mode 100644 .d4g-tools/bin/.from-scratch-sh create mode 100644 .d4g-tools/bin/d4g.env create mode 100644 .d4g-tools/bin/docker.sh create mode 100644 .d4g-tools/bin/env.sh create mode 100644 .d4g-tools/bin/fastapi/main.py create mode 100644 .d4g-tools/bin/fastapi/readme.md create mode 100644 .d4g-tools/bin/taipy/DockerFile create mode 100644 .d4g-tools/bin/taipy/compose.yml create mode 100644 .d4g-tools/bin/taipy/requirements.txt create mode 100644 .d4g-tools/lib/model/country.py create mode 100644 .github/workflows/pre-commit copy.yaml diff --git a/.d4g-tools/bin/.from-scratch-sh b/.d4g-tools/bin/.from-scratch-sh new file mode 100644 index 0000000..a0f3fae --- /dev/null +++ b/.d4g-tools/bin/.from-scratch-sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1091 +# shellcheck disable=SC1090 +# shellcheck disable=SC2034 + +set -Eeuo pipefail + +# IMPORTANT AND NECESSARY: Load dependencies +source "$LIB_DIR"/common.sh + +usage() { + cat </dev/null && pwd -P) + + while :; do + case "${1-}" in + -h | --help) + usage + ;; + -v | --verbose) + DEBUG="true" + ;; + --dummy-flag*) + DUMMY_FLAG="true" + ;; + --dummy-param=*) + DUMMY_PARAM="${1#*=}" + ;; + -?*) + echo "Unknown option: $1" + usage + ;; + *) + break + ;; + esac + shift + done + + return 0 +} + +parse_params "$@" + +echo -n "Ready to rumble." diff --git a/.d4g-tools/bin/d4g.env b/.d4g-tools/bin/d4g.env new file mode 100644 index 0000000..fc379f5 --- /dev/null +++ b/.d4g-tools/bin/d4g.env @@ -0,0 +1 @@ +#TODO ? diff --git a/.d4g-tools/bin/docker.sh b/.d4g-tools/bin/docker.sh new file mode 100644 index 0000000..7df16da --- /dev/null +++ b/.d4g-tools/bin/docker.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1091 +# shellcheck disable=SC1090 +# shellcheck disable=SC2034 + +set -Eeuo pipefail + +# IMPORTANT AND NECESSARY: Load dependencies +source "$LIB_DIR"/common.sh + +usage() { + cat </dev/null && pwd -P) + + while :; do + case "${1-}" in + -h | --help) + usage + ;; + -v | --verbose) + DEBUG="true" + ;; + --dummy-flag*) + DUMMY_FLAG="true" + ;; + --dummy-param=*) + DUMMY_PARAM="${1#*=}" + ;; + -?*) + echo "Unknown option: $1" + usage + ;; + *) + break + ;; + esac + shift + done + + return 0 +} + +parse_params "$@" + +if ! command_exists "docker"; then + brew install docker +fi + +echo -n "Ready to rumble." diff --git a/.d4g-tools/bin/env.sh b/.d4g-tools/bin/env.sh new file mode 100644 index 0000000..bc61ea8 --- /dev/null +++ b/.d4g-tools/bin/env.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1091 +# shellcheck disable=SC1090 +# shellcheck disable=SC2034 + +set -Eeuo pipefail + +# IMPORTANT AND NECESSARY: Load dependencies +source "$LIB_DIR"/common.sh + +# This script replaces all env vars defined in .env and creates .env-full + +# Specify the path to the .env file +env_file=".env" + +# Specify the path to the .dock-env file +env_clean_file=".env-clean" + +# Specify the path to the .env-full file +env_full_file=".env-full" + +# Remove the existing .env-full file if it exists +if [ -f "$env_full_file" ]; then + rm "$env_full_file" +fi + +# Remove comments and blank lines from the original .env file and write to the new .dock-env file +sed '/^[[:blank:]]*#/d; /^[[:blank:]]*$/d' "$env_file" >"$env_clean_file" + +#echo "New $env_clean_file file created without comments." + +# Set the allexport option +set -o allexport + +# Source the $env_clean_file file to load the environment variables +source "$env_clean_file" + +# Unset the allexport option +set +o allexport + +# Loop through the lines of the .env-clean file and write their values to the .env-full file +while IFS= read -r line; do + var_name=$(echo "$line" | cut -d= -f1) + var_value="${!var_name}" + echo "$var_name=$var_value" >>"$env_full_file" +done <"$env_clean_file" + +echo "--- Content of .env-full ---" +cat "$env_full_file" + +# Remove the existing $env_clean_file file if it exists +if [ -f "$env_clean_file" ]; then + rm "$env_clean_file" +fi diff --git a/.d4g-tools/bin/fastapi/main.py b/.d4g-tools/bin/fastapi/main.py new file mode 100644 index 0000000..072688d --- /dev/null +++ b/.d4g-tools/bin/fastapi/main.py @@ -0,0 +1,29 @@ +from fastapi import FastAPI, Depends +from sqlmodel import Session, SQLModel, create_engine +from sqlalchemy.orm import Session as SQLAlchemySession +import requests + +DATABASE_URL = "postgresql://user:password@localhost/database" + +engine = create_engine(DATABASE_URL) + +def get_db() -> SQLAlchemySession: + with Session(engine) as session: + yield session + +app = FastAPI() + +@app.on_event("startup") +def on_startup(): + with Session(engine) as session: + SQLModel.metadata.create_all(engine) + # Fetch data from Gapminder API + response = requests.get("https://api.gapminder.org/iso_codes") + data = response.json() + for item in data: + country = Country(name=item["name"], iso_code=item["iso_code"], region_id=item["region_id"]) + session.add(country) + session.commit() + +country_route = BaseRoute(Country, get_db) +app.include_router(country_route.router, prefix="/countries", tags=["countries"]) diff --git a/.d4g-tools/bin/fastapi/readme.md b/.d4g-tools/bin/fastapi/readme.md new file mode 100644 index 0000000..554a103 --- /dev/null +++ b/.d4g-tools/bin/fastapi/readme.md @@ -0,0 +1,160 @@ +Sure, here's a step-by-step guide to build the Flask web application as per your requirements: + +### Step 1: Scrape Data and Store Locally + +First, you'll need to scrape data from `data.gouv.fr` and store it locally as a CSV file. + +```python +import requests +import pandas as pd + +# Make a GET request to the website +url = 'https://www.data.gouv.fr/path_to_csv_file' +r = requests.get(url) + +# Save the content in a pandas DataFrame +df = pd.read_csv(r.content) + +# Save the DataFrame to a CSV file +df.to_csv('./data/initial/data.csv', index=False) +``` + +### Step 2: Create SQLModel and Persist to PostgreSQL + +Next, create a SQLModel based on the CSV headers and persist it to PostgreSQL. + +```python +from sqlmodel import SQLModel, Session, create_engine, Field +from typing import Optional +import pandas as pd + +# Load the CSV file +df = pd.read_csv('./data/initial/data.csv') + +# Define your SQLModel dynamically based on the CSV headers +class Data(SQLModel, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + for col in df.columns: + exec(f"{col} = Field(None)") + +# Create an engine and a session +engine = create_engine("postgresql://user:password@db/dbname") +SQLModel.metadata.create_all(engine) +session = Session(engine) + +# Persist the data to PostgreSQL +for index, row in df.iterrows(): + data = Data(**row.to_dict()) + session.add(data) + +session.commit() +``` + +### Step 3: Build Flask Web App + +Now, build a Flask web app to display the data using Flask-Table for enhanced table components. + +```python +from flask import Flask, render_template +from flask_table import Table, Col +from sqlmodel import select + +# Declare your table +class DataTable(Table): + id = Col('ID') + for col in df.columns: + exec(f"{col} = Col('{col}')") + +app = Flask(__name__) + +@app.route('/') +def index(): + result = session.exec(select(Data)).all() + table = DataTable(result) + return render_template('index.html', table=table) + +if __name__ == '__main__': + app.run(debug=True) +``` + +### Step 4: Dockerize the Application + +Create a `Dockerfile` for the Flask application: + +```Dockerfile +FROM python:3.8-slim + +WORKDIR /app + +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt + +COPY . . + +ENTRYPOINT ["./entrypoint.sh"] +``` + +Create an `entrypoint.sh` script: + +```sh +#!/bin/sh + +# Wait for PostgreSQL to be ready +while ! nc -z db 5432; do + sleep 1 +done + +# Run the application +exec "$@" +``` + +Make sure to give execute permission to `entrypoint.sh`: + +```sh +chmod +x entrypoint.sh +``` + +Create a `docker-compose.yml` file: + +```yaml +version: '3' +services: + web: + build: . + command: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app --bind 0.0.0.0:5000 + ports: + - "5000:5000" + depends_on: + - db + db: + image: "postgres" + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: dbname +``` + +Create a `requirements.txt` file: + +``` +Flask +Flask-Table +pandas +requests +sqlmodel +psycopg2-binary +gunicorn +uvicorn +``` + +### Step 5: Build and Run the Application + +Build and run the Docker containers: + +```sh +docker compose up --build +``` + +This will build the Docker images and start the containers. You can access the Flask web app at `http://localhost:5000`. + +Feel free to adjust the code to fit your specific needs and ensure that all dependencies are installed correctly. diff --git a/.d4g-tools/bin/python/pyproject.toml.dist b/.d4g-tools/bin/python/pyproject.toml.dist index e23e3b4..04fabb0 100644 --- a/.d4g-tools/bin/python/pyproject.toml.dist +++ b/.d4g-tools/bin/python/pyproject.toml.dist @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "$PROJECT_NAME" -version = "0.1.0" +version = "$PROJECT_VERSION" description = "$PROJECT_DESCRIPTION" authors = ["DataForGood", "$PROJECT_AUTHORS"] license = " MIT" diff --git a/.d4g-tools/bin/taipy/DockerFile b/.d4g-tools/bin/taipy/DockerFile new file mode 100644 index 0000000..adc5f99 --- /dev/null +++ b/.d4g-tools/bin/taipy/DockerFile @@ -0,0 +1,20 @@ +# Use an official Python runtime as a parent image +FROM python:3.10-slim-buster + +# Set the working directory in the container +WORKDIR /app + +# Copy the requirements file and install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the rest of the application code +ARG APP_DIR +COPY ${APP_DIR} . + +# Expose the port the app will run on +ARG PORT +EXPOSE ${PORT} + +# Run the uWSGI server +CMD ["uwsgi", "--http", "0.0.0.0:${PORT}", "--wsgi-file", "${APP_DIR}/app.py", "--callable", "app", "--master", "--processes", "4", "--threads", "2", "--logto", "/dev/stdout"] diff --git a/.d4g-tools/bin/taipy/compose.yml b/.d4g-tools/bin/taipy/compose.yml new file mode 100644 index 0000000..296ff8f --- /dev/null +++ b/.d4g-tools/bin/taipy/compose.yml @@ -0,0 +1,19 @@ +version: "3.8" + +services: + uwsgi_app: + image: python:3.9-slim-buster + working_dir: /app + volumes: + - ${APP_DIR}:/app + - ./uwsgi.ini:/app/uwsgi.ini + command: ["uwsgi", "--ini", "/app/uwsgi.ini"] + + nginx: + image: nginx:latest + ports: + - "${HTTP_PORT}:80" + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf + depends_on: + - uwsgi_app diff --git a/.d4g-tools/bin/taipy/requirements.txt b/.d4g-tools/bin/taipy/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/.d4g-tools/bin/uwsgi/uwsgi.sh b/.d4g-tools/bin/uwsgi/uwsgi.sh index 7df1b9d..fd867f5 100755 --- a/.d4g-tools/bin/uwsgi/uwsgi.sh +++ b/.d4g-tools/bin/uwsgi/uwsgi.sh @@ -40,7 +40,6 @@ After=syslog.target [Service] ExecStart=$VENV_DIR/bin/uwsgi --ini $BIN_DIR/uwsgi/uwsgi.ini:$STAGE --socket /tmp/$PROJECT_NAME.$STAGE.uwsgi.sock --module $PROJECT_NAME.main:web_app -# ExecStart=$VENV_DIR/bin/uwsgi --ini $BIN_DIR/uwsgi/uwsgi.ini:$STAGE --http-socket :$APP_PORT WorkingDirectory=$PROJECT_DIR Restart=always KillSignal=SIGINT @@ -73,7 +72,7 @@ if [ "$PROD" == "false" ]; then echo """ server { listen $HTTP_PORT; - server_name localhost; + server_name $DOMAIN; location / { include uwsgi_params diff --git a/.d4g-tools/lib/depends.sh b/.d4g-tools/lib/depends.sh index c3f31a9..8ca82a5 100755 --- a/.d4g-tools/lib/depends.sh +++ b/.d4g-tools/lib/depends.sh @@ -38,6 +38,7 @@ if [ "$(echo "$NODE_VERSION" | cut -c 2-3)" -lt 18 ]; then exit 1 fi +# TODO brew?? # Install brew if not installed. # We will use brew to install dependencies on macOS and linux if ! command -v brew &>/dev/null; then diff --git a/.d4g-tools/lib/model/country.py b/.d4g-tools/lib/model/country.py new file mode 100644 index 0000000..3140237 --- /dev/null +++ b/.d4g-tools/lib/model/country.py @@ -0,0 +1,16 @@ +from sqlmodel import SQLModel, Field +from typing import Optional + +class RegionBase(SQLModel): + name: str + +class Region(RegionBase, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + +class CountryBase(SQLModel): + name: str + iso_code: str + region_id: int + +class Country(CountryBase, table=True): + id: Optional[int] = Field(default=None, primary_key=True) diff --git a/.github/workflows/pre-commit copy.yaml b/.github/workflows/pre-commit copy.yaml new file mode 100644 index 0000000..b7e4de9 --- /dev/null +++ b/.github/workflows/pre-commit copy.yaml @@ -0,0 +1,25 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + + - name: Install poetry + run: | + python -m pip install poetry + poetry export --with dev --format=requirements.txt --output=requirements.txt + - name: set PY + run: echo "PY=$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" >> $GITHUB_ENV + - uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} + - uses: pre-commit/action@v3.0.0 diff --git a/d4g.ini b/d4g.ini index fc9ac9e..2116183 100644 --- a/d4g.ini +++ b/d4g.ini @@ -1,9 +1,10 @@ [default] -# PROJECT_NAME: WITHOUT SPACES!! + # Will be used as home directory for your project # If platform is using python, it will be used as package name in pip # example: taxplorer, bloom, genai, carbon-bombs, ..., project-for-good +# PROJECT_NAME: WITHOUT SPACES!! PROJECT_NAME = "taxplorer" ################################################ @@ -14,8 +15,8 @@ APP_VAR = "web_app" # SERVER/PLATFORM # Platform/Server definition -# example: taipy, streamlit, vercel -PLATFORM = "taipy" +# example: python (default), taipy, flask, dash, streamlit, vercel +PLATFORM = "python" [dev] ; PLATFORM = "taipy"