Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

added django management script for data imports #3

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 145 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version

# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock

# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# sqlite
*.sqlite3

/static/

db.sqlite3
Empty file added myapp/management/__init__.py
Empty file.
92 changes: 92 additions & 0 deletions myapp/management/commands/migrate_agency_site.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
from django.core.management.base import BaseCommand
from county_project.settings import BASE_DIR

from myapp.models import Agency, SitePart
import csv
import os


class Command(BaseCommand):
help = 'Migrate Site and Agency'

def handle(self, *args, **options):
self.stdout.write(self.style.SUCCESS('Importing data...'))
base_csv_path = os.getcwd() + '/templates/'
agency_csv_file = base_csv_path + '/Example School Agency Num 3.csv'
sitepart_csv_file = base_csv_path + '/Site_Part_example_csv.csv'

# Saving Agency
with open(agency_csv_file) as csv_file:
agency_csv_reader = csv.reader(csv_file, delimiter=',')
line_count = 0
for row in agency_csv_reader:
# ['system_name', 'county', 'state', 'active', 'system_type', 'address', 'city', 'zipcode', 'system_no']
if line_count == 0:
system_name_index = row.index('system_name')
county_index = row.index('county')
state_index = row.index('state')
active_index = row.index('active')
system_type_index = row.index('system_type')
address_index = row.index('address')
city_index = row.index('city')
zipcode_index = row.index('zipcode')
system_no_index = row.index('system_no')
else:
system_name = row[system_name_index]
county = row[county_index]
state = row[state_index]
active = row[active_index]
system_type = row[system_type_index]
address = row[address_index]
city = row[city_index]
zipcode = row[zipcode_index]
system_no = row[system_no_index]

agency = Agency()
agency.system_name = system_name
agency.county = county
agency.state = state
agency.active = True if int(active) == 1 else False
agency.system_type = system_type
agency.address = address
agency.city = city
agency.zipcode = zipcode
agency.system_no = system_no
agency.save()

line_count += 1

# Saving Site Part
with open(sitepart_csv_file) as csv_file:
sitepart_csv_reader = csv.reader(csv_file, delimiter=',')
line_count = 0
for row in sitepart_csv_reader:
# ['system_no', 'part_name', 'status', 'sys_site_n']
if line_count == 0:
system_no_index = row.index('system_no')
part_name_index = row.index('part_name')
status_index = row.index('status')
sys_site_n_index = row.index('sys_site_n')
else:
system_no = row[system_no_index]
part_name = row[part_name_index]
status = row[status_index]
sys_site_n = row[sys_site_n_index]

# check if agency system no exists
agency = Agency.objects.filter(system_no=system_no)
if agency.exists():
site_part = SitePart()
site_part.system_no = agency.first()
site_part.part_name = part_name
site_part.status = status
site_part.sys_site_n = sys_site_n
site_part.save()
else:
message = 'Agency with system_no {} does not exists!'.format(system_no)
self.stdout.write(self.style.ERROR(message))
line_count += 1


self.stdout.write(self.style.SUCCESS(
'Data has been successfully imported.'))
46 changes: 46 additions & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
argon2-cffi==20.1.0
asgiref==3.3.1
astroid==2.4.2
babelfish==0.5.5
bcrypt==3.2.0
beautifulsoup4==4.9.3
certifi==2020.11.8
cffi==1.14.5
chardet==3.0.4
colorama==0.4.4
defusedxml==0.7.1
diff-match-patch==20200713
Django==3.1.7
django-bootstrap-v5==1.0.2
django-crispy-forms==1.11.2
django-debug-toolbar==3.2
django-extensions==3.1.2
django-import-export==2.5.0
django-object-tools==2.0.0
django-phone-field==1.8.1
et-xmlfile==1.0.1
idna==2.10
isort==5.6.4
jdcal==1.4.1
lazy-object-proxy==1.4.3
MarkupPy==1.14
mccabe==0.6.1
numpy==1.19.4
odfpy==1.4.1
openpyxl==3.0.6
pandas==1.1.4
pycparser==2.20
pylint==2.6.0
python-dateutil==2.8.1
pytz==2020.4
PyYAML==5.4.1
requests==2.25.0
six==1.15.0
soupsieve==2.2.1
sqlparse==0.4.1
tablib==3.0.0
toml==0.10.2
urllib3==1.26.2
wrapt==1.12.1
xlrd==2.0.1
xlwt==1.3.0
2 changes: 1 addition & 1 deletion templates/Example School Agency Num 3.csv
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ MULBERRY UNION SCHOOL,IMPERIAL,CA,1,ED,1391 RUTHERFORD Rd,BRAWLEY,92227,1300556
PINE UNION SCHOOL,IMPERIAL,CA,1,ED,3295 Holt Rd,HOLTVILLE,92250,1300560
MCCABE UNION SCHOOL,IMPERIAL,CA,1,ED,701 W Mc Cabe Rd,EL CENTRO,92243,1300579
ROUND VALLEY SCHOOL,INYO,CA,1,ED,"300 N. Round Valley Rd, Bishop, Ca 93514",Bishop,93514,1400019
PALISADE GLACIER HIGH SCHOOL,INYO,CA,1,ED,"P.O. Box 998, Big Pine, CA 93513",Big Pine,93513,1400522
PALISADE GLACIER HIGH SCHOOL,INYO,CA,1,ED,"P.O. Box 998, Big Pine, CA 93513",Big Pine,93513,1400522
4 changes: 2 additions & 2 deletions templates/Site_Part_example_csv.csv
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
system_no,part_name,status,sys_site_n
system_no,part_name,status,sys_site_n
1300553,MAGNOLIA SCHOOL,AC,1300553-001
1300553,Backup Generator,SB,1300553-002
1300554,MEADOWS UNION SCHOOL,AC,1300554-001
1300554,Backup Generator,SB,1300554-002
1300556,MULBERRY UNION SCHOOL,AC,1300556-001
1300560,PINE UNION SCHOOL,AC,1300560-001
1300560,Backup Generator,SB,1300560-002
1300560,Backup Generator,SB,1300560-002