From d1baae546ae6f59a0970d94ed8e69550f3f271d1 Mon Sep 17 00:00:00 2001 From: Robert Avram Date: Fri, 16 Sep 2016 15:26:38 -0400 Subject: [PATCH] R 2.2.3 (#151) * Etrips fix (#73) * apk updates * version update * apk name change * Etrips fix (#74) * apk updates * version update * apk name change * temporary change of ipa link * Etrips fix (#75) * apk updates * version update * apk name change * temporary change of ipa link * plist fix * Etrips fix (#76) * apk updates * version update * apk name change * temporary change of ipa link * plist fix * change cert * Develop (#77) * Etrips fix (#73) * apk updates * version update * apk name change * Etrips fix (#74) * apk updates * version update * apk name change * temporary change of ipa link * Etrips fix (#75) * apk updates * version update * apk name change * temporary change of ipa link * plist fix * Etrips fix (#76) * apk updates * version update * apk name change * temporary change of ipa link * plist fix * change cert * Various fixes (#82) * FIX: trip locations modified * remove print statement * allow numbers in the column headers for workplans * partner_portal fix * add picture upload api again * add permission classes * Develop (#83) * Develop (#65) (#66) * changed deprecated name for the number (#37) #118468621 * update build process (#38) * update build process [DO NOT MERGE] * build updates * small tweaks * small tweaks * get around permissions * update gitignore * display spinner when waiting for map location search results (#27) * display spinner when waiting for map location search results * simplify loading spinner * remove unneeded Icon file * fix government ntervention implementation status [Finishes #105550212] * update Android / iOS builds (#41) * set agreement start end dates based on signed dates and result structure (#34) * set agreement start end dates based on signed dates and result structure * move start end date auto population from model to form * updated error messages * added agreement form tests * added start date validation and test * fix-government-in-simple WIP [WIP] [DO NOT MERGE] * migration fixes * fix-circle-ci-build (#50) * fix-circle-ci-build updated library fail * correct spelling error * small updates * Added etools/github contribution best practices guide (#40) * Added etools/github contribution best practices guide A document introducing new developers to the eTools project workflow, and best practices when contributing to the GitHub. * docs(CONTRIBUTING): change file name * docs(guide): add commit guidelines section Add section on committing message best practices Add section headers dividing concept from practices PR (#40) * small tweaks * HACT dashboard tweaks to show trips for government interventions * small tweak * Listing Trip's Action Points in email notification (#30) * Fix small issues - BY Ali (#44) * fix distribution items checkbox readonly * Fix some issues Result display name Approve trip by supervisor Do not send trip notification to PA if no budget owner * Fix test sending email to PA with budget owner * tweaks * migration add (#56) * remove existing migration * fix migration dependency * gitingore update (#57) * small tweaks * add migration IN DEV ROLL BACK TRIPS MIGRATIONS TO 16 AND RUN MIGRATIONS AGAIN * ENJOY PERFORMANCE Fix performance issue on compound queries * fix locations latency * planned cash transfers for non-gov * planned visits for NGOs * only show government partners if they have interventions * fix unicode return * formatting fix * small tweaks * refractor(templates/admin): change test environments to have more obvious headers (#53) * refractor(templates/admin): change test environments to have more obvious headers Remove "Testing Environment" top bar. Change banner in localhost, etools-dev, and etools-staging to be red and display "Local Testing Environment", "Develop Testing Environment", and "Staging Testing Environment" on banner respectively. * refractor(#header): obvious testing header for dashboard Added change from 097f25cbdfa58b4866e385b19be9aa22284c198e to etools dashboard * docs(guide): new dev setup guide for linux/ubuntu (#54) * feat(views) add outdated browser page Add javascript in `base.html` to load `outdated_browser.html` when browser is IE 10 or older (user-agent contains 'MSIE') * changed cartodb SQL statement * changed tiles layer to Esri as Mapquest has changed their api * fix(views) store browser logos on server Previously was fetching logo images from remote url's Now images are stored on server and fetched from there. * addded comment * build-fix (#64) * build-fix * remove sudo * add npm * update * two run commands * tweaks * remove sudo * remove sudo * update * update * small update * SMALL TWEAK * small tweak * Etrips fix (#73) * apk updates * version update * apk name change * Etrips fix (#74) * apk updates * version update * apk name change * temporary change of ipa link * Etrips fix (#75) * apk updates * version update * apk name change * temporary change of ipa link * plist fix * Etrips fix (#76) * apk updates * version update * apk name change * temporary change of ipa link * plist fix * change cert * Develop (#77) (#78) * Etrips fix (#73) * apk updates * version update * apk name change * Etrips fix (#74) * apk updates * version update * apk name change * temporary change of ipa link * Etrips fix (#75) * apk updates * version update * apk name change * temporary change of ipa link * plist fix * Etrips fix (#76) * apk updates * version update * apk name change * temporary change of ipa link * plist fix * change cert * Various fixes (#82) * FIX: trip locations modified * remove print statement * allow numbers in the column headers for workplans * partner_portal fix * add picture upload api again * add permission classes * fix funding and ram sync (#87) * Fix Funding Sync for Vision (#88) * fix funding and ram sync * fix funding and ram sync * fix funding sync * Fix ram sync (#89) * Etrips fix (#73) * apk updates * version update * apk name change * Etrips fix (#74) * apk updates * version update * apk name change * temporary change of ipa link * Etrips fix (#75) * apk updates * version update * apk name change * temporary change of ipa link * plist fix * Etrips fix (#76) * apk updates * version update * apk name change * temporary change of ipa link * plist fix * change cert * Various fixes (#82) * FIX: trip locations modified * remove print statement * allow numbers in the column headers for workplans * partner_portal fix * add picture upload api again * add permission classes * fix funding and ram sync (#87) * Fix Funding Sync for Vision (#88) * fix funding and ram sync * fix funding and ram sync * fix funding sync * celery flower * small tweak * celery flower (#91) * celery flower * small tweak * worker long-running tasks setting * broker visibility timeout in env (#96) * Various fixes 002 (#97) * broker visibility timeout in env * no record * updated country lat long decimal places and validation (#99) * update .gitignore to ignore pmp_frontend (#98) * Fix Agreement start end dates (#94) * set agreement start end dates based on signed dates and result structure * move start end date auto population from model to form * updated error messages * added agreement form tests * added start date validation and test * set end date for PCAs only * set end date for PCAs only * Various fixes 002 (#100) * broker visibility timeout in env * no record * locations import fix * added ability to hide partner records (#69) * Fix development setup (#90) * Fix development environment setup * Move local dependencies to requirements/local.txt * Add local_base settings so local deps can be removed when CI runs * partners model changes [MIGRATION NEEDED] * vision sync * expose total cash * partner sync tweaks * fake tasks * fix dev deploy * allow 30 seconds for time difference between adfs server and etools server * change to 5 mins * change to 5 mins * import Decimal * Develop (#108) * Fix locations map 500 error (#107) * list empty coordinates in locations * took out extra code * took out extra code * took out extra code * trip exports fix (#102) * Develop (#111) * Fix locations map 500 error (#107) * list empty coordinates in locations * took out extra code * took out extra code * took out extra code * trip exports fix (#102) * added 60secs to jwt auth leeway (#110) * Revert "Develop" (#112) * Develop (#116) * Fix locations map 500 error (#107) * list empty coordinates in locations * took out extra code * took out extra code * took out extra code * trip exports fix (#102) * added 60secs to jwt auth leeway (#110) * Feature results api (#109) * Fix development environment setup * Results Model and API changes * Downgrade Factory boy back to the original 2.5.2, to fix build * Use UserFactory for tagging users in tests to fix build * Remove explicit ID definition and use Meta fields instead * Add IsAdminUser as permission for Milestone View * Refactor sections field to M2M to users.models.Section * Add IsAdminUser persmission class to ResultViewSet, fix permission class on MilestoneViewSet * Remove unused import * Remove unused import #2 * Change status labels * Remove labels field * Add missing migration file for removed labels field * Utilize pre-loaded ResultTypes during test * Refactor milestone field from O2O to M2O * Resolve database migration conflicts * remove all migrations that could conflict * add new migrations * fix export interventions (#115) * fix broken conflict * Develop (#119) * Fix locations map 500 error (#107) * list empty coordinates in locations * took out extra code * took out extra code * took out extra code * trip exports fix (#102) * added 60secs to jwt auth leeway (#110) * Feature results api (#109) * Fix development environment setup * Results Model and API changes * Downgrade Factory boy back to the original 2.5.2, to fix build * Use UserFactory for tagging users in tests to fix build * Remove explicit ID definition and use Meta fields instead * Add IsAdminUser as permission for Milestone View * Refactor sections field to M2M to users.models.Section * Add IsAdminUser persmission class to ResultViewSet, fix permission class on MilestoneViewSet * Remove unused import * Remove unused import #2 * Change status labels * Remove labels field * Add missing migration file for removed labels field * Utilize pre-loaded ResultTypes during test * Refactor milestone field from O2O to M2O * Resolve database migration conflicts * remove all migrations that could conflict * add new migrations * fix export interventions (#115) * fix vision sync (#118) partners get total cash even if negative decimal * add util_scripts * Develop (#145) * Fix locations map 500 error (#107) * list empty coordinates in locations * took out extra code * took out extra code * took out extra code * trip exports fix (#102) * added 60secs to jwt auth leeway (#110) * Feature results api (#109) * Fix development environment setup * Results Model and API changes * Downgrade Factory boy back to the original 2.5.2, to fix build * Use UserFactory for tagging users in tests to fix build * Remove explicit ID definition and use Meta fields instead * Add IsAdminUser as permission for Milestone View * Refactor sections field to M2M to users.models.Section * Add IsAdminUser persmission class to ResultViewSet, fix permission class on MilestoneViewSet * Remove unused import * Remove unused import #2 * Change status labels * Remove labels field * Add missing migration file for removed labels field * Utilize pre-loaded ResultTypes during test * Refactor milestone field from O2O to M2O * Resolve database migration conflicts * remove all migrations that could conflict * add new migrations * fix export interventions (#115) * fix vision sync (#118) partners get total cash even if negative decimal * Add assumtions fields to Indicator model * Allow partial updates on API (#121) * feature add result type subactivity (#124) * Add new ResultType: SubActivity * Rename result type * added 60secs to jwt auth leeway (#125) good work ;) * fix Result Model (#130) * fix Result Model * fix wbs null constraint on empty strings * add migrations * small tweaks * other small tweaks * final tweaks * fix partner sync * resolve conflict * Fix - major vision sync improvements (#138) * fix Result Model * fix wbs null constraint on empty strings * add migrations * small tweaks * other small tweaks * final tweaks * fix partner sync * partner sync fixes * Funding Synchronizer fixes FCs do not duplicate anymore, FC line item expenditures are aggregated across all records with the same FC ref Performance improvements on FCs * allow more funds to be connected to trips * programme synchronizer performance and stability improvements * add clean result type script * skip bad wbs records * removed cash transfer fields from admin form (#139) renamed deleted flag * wbs skipping activity and logging skips * Fix result structure 001 nik (#140) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * remove fr number dependency on intervention * remove tests * Fix result structure 001 nik (#141) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * results cleanup of result structures change to no cascade on foreign key to result structure * remove result structure filter * cleanup util scritps * only remove duplicate ram indicators * fix ram synchronizer to check only cropped versions of fields * Migrations for result structure (#142) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * results cleanup of result structures change to no cascade on foreign key to result structure * cascade do nothing migrations * update migration * Fix result structure 001 nik (#143) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * results cleanup of result structures change to no cascade on foreign key to result structure * cascade do nothing migrations * added mixin * RAM results for inteventions * removed result_structure.name * Ram indicator migrations (#144) * fix Result Model * fix wbs null constraint on empty strings * add migrations * small tweaks * other small tweaks * final tweaks * fix partner sync * partner sync fixes * Funding Synchronizer fixes FCs do not duplicate anymore, FC line item expenditures are aggregated across all records with the same FC ref Performance improvements on FCs * allow more funds to be connected to trips * programme synchronizer performance and stability improvements * add clean result type script * removed cash transfer fields from admin form renamed deleted flag * skip bad wbs records * removed cash transfer fields from admin form (#139) renamed deleted flag * fix indicator import * wbs skipping activity and logging skips * Fix result structure 001 nik (#140) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * remove fr number dependency on intervention * remove tests * results cleanup of result structures change to no cascade on foreign key to result structure * Fix result structure 001 nik (#141) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * results cleanup of result structures change to no cascade on foreign key to result structure * remove result structure filter * cleanup util scritps * only remove duplicate ram indicators * fix ram synchronizer to check only cropped versions of fields * cascade do nothing migrations * Migrations for result structure (#142) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * results cleanup of result structures change to no cascade on foreign key to result structure * cascade do nothing migrations * update migration * added mixin * RAM results for inteventions * removed result_structure.name * updated ram indicator * Develop (#150) * Fix locations map 500 error (#107) * list empty coordinates in locations * took out extra code * took out extra code * took out extra code * trip exports fix (#102) * added 60secs to jwt auth leeway (#110) * Feature results api (#109) * Fix development environment setup * Results Model and API changes * Downgrade Factory boy back to the original 2.5.2, to fix build * Use UserFactory for tagging users in tests to fix build * Remove explicit ID definition and use Meta fields instead * Add IsAdminUser as permission for Milestone View * Refactor sections field to M2M to users.models.Section * Add IsAdminUser persmission class to ResultViewSet, fix permission class on MilestoneViewSet * Remove unused import * Remove unused import #2 * Change status labels * Remove labels field * Add missing migration file for removed labels field * Utilize pre-loaded ResultTypes during test * Refactor milestone field from O2O to M2O * Resolve database migration conflicts * remove all migrations that could conflict * add new migrations * fix export interventions (#115) * fix vision sync (#118) partners get total cash even if negative decimal * Add assumtions fields to Indicator model * Allow partial updates on API (#121) * feature add result type subactivity (#124) * Add new ResultType: SubActivity * Rename result type * added 60secs to jwt auth leeway (#125) good work ;) * fix Result Model (#130) * fix Result Model * fix wbs null constraint on empty strings * add migrations * small tweaks * other small tweaks * final tweaks * fix partner sync * resolve conflict * Fix - major vision sync improvements (#138) * fix Result Model * fix wbs null constraint on empty strings * add migrations * small tweaks * other small tweaks * final tweaks * fix partner sync * partner sync fixes * Funding Synchronizer fixes FCs do not duplicate anymore, FC line item expenditures are aggregated across all records with the same FC ref Performance improvements on FCs * allow more funds to be connected to trips * programme synchronizer performance and stability improvements * add clean result type script * skip bad wbs records * removed cash transfer fields from admin form (#139) renamed deleted flag * wbs skipping activity and logging skips * Fix result structure 001 nik (#140) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * remove fr number dependency on intervention * remove tests * Fix result structure 001 nik (#141) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * results cleanup of result structures change to no cascade on foreign key to result structure * remove result structure filter * cleanup util scritps * only remove duplicate ram indicators * fix ram synchronizer to check only cropped versions of fields * Migrations for result structure (#142) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * results cleanup of result structures change to no cascade on foreign key to result structure * cascade do nothing migrations * update migration * Fix result structure 001 nik (#143) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * results cleanup of result structures change to no cascade on foreign key to result structure * cascade do nothing migrations * added mixin * RAM results for inteventions * removed result_structure.name * Ram indicator migrations (#144) * fix Result Model * fix wbs null constraint on empty strings * add migrations * small tweaks * other small tweaks * final tweaks * fix partner sync * partner sync fixes * Funding Synchronizer fixes FCs do not duplicate anymore, FC line item expenditures are aggregated across all records with the same FC ref Performance improvements on FCs * allow more funds to be connected to trips * programme synchronizer performance and stability improvements * add clean result type script * removed cash transfer fields from admin form renamed deleted flag * skip bad wbs records * removed cash transfer fields from admin form (#139) renamed deleted flag * fix indicator import * wbs skipping activity and logging skips * Fix result structure 001 nik (#140) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * remove fr number dependency on intervention * remove tests * results cleanup of result structures change to no cascade on foreign key to result structure * Fix result structure 001 nik (#141) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * results cleanup of result structures change to no cascade on foreign key to result structure * remove result structure filter * cleanup util scritps * only remove duplicate ram indicators * fix ram synchronizer to check only cropped versions of fields * cascade do nothing migrations * Migrations for result structure (#142) * removed cash transfer fields from admin form renamed deleted flag * fix indicator import * results cleanup of result structures change to no cascade on foreign key to result structure * cascade do nothing migrations * update migration * added mixin * RAM results for inteventions * removed result_structure.name * updated ram indicator * temporarily remove hact dashboard (#149) * temporarily remove hact dashboard remove dashboard until fixed * add a print statement to keep the console alive --- EquiTrack/EquiTrack/urls.py | 2 +- EquiTrack/EquiTrack/util_scripts.py | 382 ++++++++++++++++++ .../migrations/0004_auto_20160909_2258.py | 18 + .../migrations/0005_auto_20160910_0836.py | 18 + EquiTrack/funds/models.py | 8 +- EquiTrack/partners/admin.py | 22 +- .../migrations/0067_auto_20160910_0836.py | 24 ++ ...8_remove_fundingcommitment_intervention.py | 18 + .../migrations/0069_auto_20160915_2222.py | 25 ++ .../migrations/0070_auto_20160915_2340.py | 20 + EquiTrack/partners/mixins.py | 15 +- EquiTrack/partners/models.py | 24 +- .../templatetags/intervention_tags.py | 2 +- EquiTrack/partners/tests/test_models.py | 15 +- EquiTrack/reports/admin.py | 12 +- EquiTrack/reports/fixtures/initial_data.json | 37 +- .../migrations/0020_indicator_assumptions.py | 19 + .../migrations/0021_auto_20160906_1925.py | 48 +++ .../migrations/0022_auto_20160906_1927.py | 26 ++ .../migrations/0023_auto_20160909_2203.py | 19 + .../migrations/0024_auto_20160915_2222.py | 30 ++ EquiTrack/reports/models.py | 64 ++- EquiTrack/reports/tests/test_views.py | 23 ++ EquiTrack/reports/views.py | 9 +- EquiTrack/templates/base.html | 6 +- EquiTrack/trips/admin.py | 15 +- EquiTrack/vision/adapters/funding.py | 104 +++-- EquiTrack/vision/adapters/partner.py | 251 +++++++++--- EquiTrack/vision/adapters/programme.py | 223 ++++++++-- EquiTrack/vision/utils.py | 8 +- 30 files changed, 1279 insertions(+), 208 deletions(-) create mode 100644 EquiTrack/EquiTrack/util_scripts.py create mode 100644 EquiTrack/funds/migrations/0004_auto_20160909_2258.py create mode 100644 EquiTrack/funds/migrations/0005_auto_20160910_0836.py create mode 100644 EquiTrack/partners/migrations/0067_auto_20160910_0836.py create mode 100644 EquiTrack/partners/migrations/0068_remove_fundingcommitment_intervention.py create mode 100644 EquiTrack/partners/migrations/0069_auto_20160915_2222.py create mode 100644 EquiTrack/partners/migrations/0070_auto_20160915_2340.py create mode 100644 EquiTrack/reports/migrations/0020_indicator_assumptions.py create mode 100644 EquiTrack/reports/migrations/0021_auto_20160906_1925.py create mode 100644 EquiTrack/reports/migrations/0022_auto_20160906_1927.py create mode 100644 EquiTrack/reports/migrations/0023_auto_20160909_2203.py create mode 100644 EquiTrack/reports/migrations/0024_auto_20160915_2222.py diff --git a/EquiTrack/EquiTrack/urls.py b/EquiTrack/EquiTrack/urls.py index 1f18f3086b..3275581ee4 100644 --- a/EquiTrack/EquiTrack/urls.py +++ b/EquiTrack/EquiTrack/urls.py @@ -105,7 +105,7 @@ url(r'^partnerships', login_required(PartnershipsView.as_view()), name='partnerships_dashboard'), url(r'^map/$', login_required(MapView.as_view()), name='map'), url(r'^cmt/$', login_required(CmtDashboardView.as_view()), name='cmt'), - url(r'^hact/$', login_required(HACTDashboardView.as_view()), name='hact_dashboard'), + #url(r'^hact/$', login_required(HACTDashboardView.as_view()), name='hact_dashboard'), url(r'^locations/', include('locations.urls')), url(r'^management/', include('management.urls')), diff --git a/EquiTrack/EquiTrack/util_scripts.py b/EquiTrack/EquiTrack/util_scripts.py new file mode 100644 index 0000000000..88cf6d0a3b --- /dev/null +++ b/EquiTrack/EquiTrack/util_scripts.py @@ -0,0 +1,382 @@ +from __future__ import print_function +from django.db import connection +from django.db.models import Count +import time +from datetime import datetime, timedelta +from users.models import Country +from reports.models import ResultType, Result, CountryProgramme, Indicator, ResultStructure +from partners.models import FundingCommitment + +def printtf(*args): + print([arg for arg in args]) + f = open('mylogs.txt','a') + print([arg for arg in args], file=f) + f.close() + +def set_country(name): + connection.set_tenant(Country.objects.get(name=name)) + + +def fix_duplicate_indicators(country_name): + if not country_name: + printtf("country name required /n") + set_country(country_name) + printtf("Fixing duplicate indicators for {}".format(country_name)) + fattrs = ["ramindicator_set"] + + def fix_indicator_code(): + printtf('cleaning codes of indicators for country ', country_name) + indicators = Indicator.objects.filter(code__exact='').all() + for ind in indicators: + ind.code = None + ind.save() + time.sleep(3) + fix_indicator_code() + + def relates_to_anything(cobj): + for a in fattrs: + if getattr(cobj, a).count(): + printtf(cobj.id, cobj, "relates to ", a) + return True + return False + def update_relationships(dpres, keep): + for a in fattrs: + objs = getattr(dpres, a).all() + if len(objs): + for obj in objs: + obj.indicator = keep + obj.save() + printtf("saved obj.id={} obj {} with keepid{} keep {}".format(obj.id, obj, keep.id, keep)) + + def _run_clean(dupes): + printtf(len(dupes), dupes) + for dup in dupes: + dupresults = Indicator.objects.filter(code=dup['code'], result=dup['result']).all() + delq = [] + keep = None + for dpres in dupresults: + if not keep: + keep = dpres + continue + else: + error = False + if relates_to_anything(dpres): + try: + update_relationships(dpres, keep) + except Exception as exp: + printtf('Cannot remove Object {}, id={}'.format(dpres, dpres.id)) + error = True + if error: + printtf("ERROR OCCURED ON RECORD", dpres.id, dpres) + continue + delq.append(dpres) + if not len(delq): + printtf("Nothing is getting removed for {}".format(dupes)) + else: + # delete everyting in the queue + [i.delete() for i in delq] + printtf("deleting: ", delq) + + dupes = Indicator.objects.values('code', 'result').order_by('code', 'result').annotate(Count('pk')).filter(pk__count__gt=1, ram_indicator=True).all() + _run_clean(dupes) + +def fix_duplicate_results(country_name): + + if not country_name: + printtf("country name required /n") + set_country(country_name) + printtf("Fixing duplicate Results for {}".format(country_name)) + # foreign attributes + fattrs = ["governmentinterventionresult_set", + "indicator_set", + "ramindicator_set", + "resultchain_set", + "tripfunds_set"] + fattrs_mapping = { + "governmentinterventionresult_set": "result", + "indicator_set" : "result", + "ramindicator_set": "result", + "resultchain_set": "result", + "tripfunds_set": "wbs" + } + def fix_string_wbs(): + results = Result.objects.filter(wbs__exact='').all() + for res in results: + res.wbs = None + res.save() + fix_string_wbs() + + def reparent_children(current_object, new_parent): + for child in current_object.get_children(): + child.parent = new_parent + printtf( "reparenting child", child.id, child, new_parent.id, new_parent) + child.save() + def relates_to_anything(cobj): + for a in fattrs: + if getattr(cobj, a).count(): + printtf(cobj.id, cobj, "relates to ", a) + return True + return False + def update_relationships(dpres, keep): + for a in fattrs: + objs = getattr(dpres, a).all() + if len(objs): + for obj in objs: + if a == "tripfunds_set": + obj.wbs = keep + else: + obj.result = keep + obj.save() + printtf("saved obj.id={} obj {} with keepid{} keep {}".format(obj.id, obj, keep.id, keep)) + def _run_clean(prop, dupes): + printtf(len(dupes), dupes) + search_string = prop + '__exact' + for dup in dupes: + dupresults = Result.objects.prefetch_related('result_type').filter(**{search_string: dup[prop]}).all() + delq = [] + keep = None + for dpres in dupresults: + if not keep: + keep = dpres + continue + else: + error = False + if dpres.get_children(): + try: + reparent_children(dpres, keep) + except Exception as exp: + printtf('Cannot reparent from Object {}, id={}'.format(dpres, dpres.id)) + error = True + if relates_to_anything(dpres): + try: + update_relationships(dpres, keep) + except Exception as exp: + printtf('Cannot remove Object {}, id={}'.format(dpres, dpres.id)) + error = True + if error: + printtf("ERROR OCCURED ON RECORD", dpres.id, dpres) + continue + delq.append(dpres) + time.sleep(0.3) + if not len(delq): + printtf("Nothing is getting removed for {}".format(dupes)) + else: + # delete everyting in the queue + [i.delete() for i in delq] + printtf("deleting: ", delq) + # get all duplicates that have the same wbs + dupes = Result.objects.values('wbs').annotate(Count('wbs')).order_by().filter(wbs__count__gt=1, wbs__isnull=False).exclude(wbs__exact='').all() + dupes = sorted(dupes, key=lambda x: x['wbs'].count('/'), reverse=True) + _run_clean('wbs', dupes) + +def cp_fix(country_name): + if not country_name: + printtf("country name required /n") + set_country(country_name) + printtf("Fixing Country Programme for {}".format(country_name)) + def get_cpwbs(wbs): + grp = wbs.split('/') + return '/'.join(grp[:3]) + + results = Result.objects.filter(wbs__isnull=False).exclude(wbs__exact='') + locpwbs = [] + for res in results: + cpwbs = get_cpwbs(res.wbs) + if cpwbs not in locpwbs: + locpwbs.append(cpwbs) + + today = datetime.now() + for i in range(len(locpwbs)): + today = today + timedelta(days=i) + tomorrow = today + timedelta(days=365) + cp, created = CountryProgramme.objects.get_or_create(wbs=locpwbs[i], name='Country Programme '+str(i), from_date=today, to_date=tomorrow) + + time.sleep(5) + + for res in results: + cp = CountryProgramme.objects.get(wbs=get_cpwbs(res.wbs)) + res.country_programme = cp + res.save() + print(res.name) + +def clean_result_types(country_name): + if not country_name: + printtf("country name required /n") + set_country(country_name) + if not country_name: + printtf("country name required /n") + + set_country(country_name) + printtf("Fixing duplicate Result Types for {}".format(country_name)) + fattrs = ["result_set"] + + def relates_to_anything(cobj): + for a in fattrs: + if getattr(cobj, a).count(): + printtf(cobj.id, cobj, "relates to ", a) + return True + return False + + def update_relationships(dpres, keep): + for a in fattrs: + objs = getattr(dpres, a).all() + if len(objs): + for obj in objs: + obj.result_type = keep + obj.save() + printtf("saved obj.id={} obj {} with keepid{} keep {}".format(obj.id, obj, keep.id, keep)) + + def _run_clean(dupes): + printtf(len(dupes), dupes) + for dup in dupes: + dupresults = ResultType.objects.filter(name=dup['name']).all() + delq = [] + keep = None + for dpres in dupresults: + if not keep: + keep = dpres + continue + else: + error = False + if relates_to_anything(dpres): + try: + update_relationships(dpres, keep) + except Exception as exp: + printtf('Cannot remove Object {}, id={}'.format(dpres, dpres.id)) + error = True + if error: + printtf("ERROR OCCURED ON RECORD", dpres.id, dpres) + continue + delq.append(dpres) + if not len(delq): + printtf("Nothing is getting removed for {}".format(dupes)) + else: + # delete everyting in the queue + [i.delete() for i in delq] + printtf("deleting: ", delq) + + # get all duplicates that have the same wbs + dupes = ResultType.objects.values('name').annotate(Count('name')).order_by().filter(name__count__gt=1).all() + _run_clean(dupes) + +def clean_result_structures(country_name): + if not country_name: + printtf("country name required /n") + set_country(country_name) + if not country_name: + printtf("country name required /n") + + set_country(country_name) + printtf("Fixing duplicate Result Structures for {}".format(country_name)) + fattrs = ["result_set", + "indicator_set", + "goal_set", + "pca_set", + "governmentintervention_set", ] + + def relates_to_anything(cobj): + for a in fattrs: + if getattr(cobj, a).count(): + printtf(cobj.id, cobj, "relates to ", a) + return True + return False + + def update_relationships(dpres, keep): + for a in fattrs: + objs = getattr(dpres, a).all() + if len(objs): + for obj in objs: + obj.result_structure = keep + obj.save() + printtf("saved obj.id={} obj {} with keepid{} keep {}".format(obj.id, obj, keep.id, keep)) + + def _run_clean(dupes): + printtf(len(dupes), dupes) + for dup in dupes: + dupresults = ResultStructure.objects.filter(name=dup['name']).all() + delq = [] + keep = None + for dpres in dupresults: + if not keep: + keep = dpres + continue + else: + error = False + if relates_to_anything(dpres): + try: + update_relationships(dpres, keep) + except Exception as exp: + printtf('Cannot remove Object {}, id={}'.format(dpres, dpres.id)) + error = True + if error: + printtf("ERROR OCCURED ON RECORD", dpres.id, dpres) + continue + delq.append(dpres) + if not len(delq): + printtf("Nothing is getting removed for {}".format(dupes)) + else: + # delete everyting in the queue + [i.delete() for i in delq] + printtf("deleting: ", delq) + + # get all duplicates that have the same name + dupes = ResultStructure.objects.values('name', 'from_date', 'to_date').order_by('name', 'from_date', 'to_date').annotate(Count('pk')).filter(pk__count__gt=1).all() + _run_clean(dupes) + + +def delete_all_fcs(country_name): + if not country_name: + printtf("country name required /n") + set_country(country_name) + printtf("Deleting all FCs for {}".format(country_name)) + fcs = FundingCommitment.objects.all() + fcs.delete() + +def dissasociate_result_structures(country_name): + if not country_name: + printtf("country name required /n") + set_country(country_name) + printtf("Dissasociating result structures for {}".format(country_name)) + results = Result.objects.all() + for result in results: + if result.wbs and result.result_structure: + result.result_structure = None + result.save() + + +def all_countries_do(function, name): + for cntry in Country.objects.order_by('name').all(): + if cntry.name in ['Global']: + continue + printtf("CALLING {} for all countries".format(name)) + function(cntry.name) + + + +def before_code_merge(): + # Clean results + all_countries_do(fix_duplicate_results, 'Result Cleaning') + + # Clean results structure + all_countries_do(clean_result_structures, 'Result Structure Cleaning') + + # clean result types + all_countries_do(clean_result_types, 'Result Types Cleaning') + + # Clean indicators + all_countries_do(fix_duplicate_indicators, 'Indicators Cleaning') + + # Delete all fcs + all_countries_do(delete_all_fcs, 'Deleting FCs') + + print('FINISHED WITH BEFORE MERGE') + +def after_code_merge(): #and after migrations + + # set up country programme + all_countries_do(cp_fix, 'Country Programme setup') + + # disassociate result structures + all_countries_do(dissasociate_result_structures, 'Dissasociate Result Structure') + + print("don't forget to sync") diff --git a/EquiTrack/funds/migrations/0004_auto_20160909_2258.py b/EquiTrack/funds/migrations/0004_auto_20160909_2258.py new file mode 100644 index 0000000000..b501802852 --- /dev/null +++ b/EquiTrack/funds/migrations/0004_auto_20160909_2258.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('funds', '0003_grant_description'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='grant', + unique_together=set([('donor', 'name')]), + ), + ] diff --git a/EquiTrack/funds/migrations/0005_auto_20160910_0836.py b/EquiTrack/funds/migrations/0005_auto_20160910_0836.py new file mode 100644 index 0000000000..1da59e44d2 --- /dev/null +++ b/EquiTrack/funds/migrations/0005_auto_20160910_0836.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('funds', '0004_auto_20160909_2258'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='grant', + unique_together=set([]), + ), + ] diff --git a/EquiTrack/funds/models.py b/EquiTrack/funds/models.py index 59359e3b19..7c4dfbfa52 100644 --- a/EquiTrack/funds/models.py +++ b/EquiTrack/funds/models.py @@ -19,6 +19,10 @@ def get_queryset(self): return super(GrantManager, self).get_queryset().select_related('donor') +class GrantManager(models.Manager): + def get_queryset(self): + return super(GrantManager, self).get_queryset().select_related('donor') + class Grant(models.Model): donor = models.ForeignKey(Donor) @@ -26,11 +30,11 @@ class Grant(models.Model): description = models.CharField(max_length=255, null=True, blank=True) expiry = models.DateField(null=True, blank=True) + objects = GrantManager() + class Meta: ordering = ['donor'] - objects = GrantManager() - def __unicode__(self): return u"{}: {}".format( self.donor.name, diff --git a/EquiTrack/partners/admin.py b/EquiTrack/partners/admin.py index cc984feae5..2369e7d6af 100644 --- a/EquiTrack/partners/admin.py +++ b/EquiTrack/partners/admin.py @@ -56,7 +56,7 @@ PCAGrantFilter, PCAGatewayTypeFilter, ) -from .mixins import ReadOnlyMixin, SectorMixin +from .mixins import ReadOnlyMixin, SectorMixin, HiddenPartnerMixin from .forms import ( PartnershipForm, PartnersAdminForm, @@ -198,16 +198,15 @@ class LinksInlineAdmin(GenericLinkStackedInline): class IndicatorsInlineAdmin(ReadOnlyMixin, admin.TabularInline): suit_classes = u'suit-tab suit-tab-results' model = RAMIndicator + verbose_name = 'RAM Result' + verbose_name_plural = 'RAM Results' extra = 1 - readonly_fields = ( - u'baseline', - u'target' - ) + fields = ('result',) def formfield_for_foreignkey(self, db_field, request=None, **kwargs): if db_field.name == u'result': - kwargs['queryset'] = Result.objects.filter(result_type__name=u'Output', ram=True) + kwargs['queryset'] = Result.objects.filter(result_type__name=u'Output', ram=True, hidden=False) return super(IndicatorsInlineAdmin, self).formfield_for_foreignkey( db_field, request, **kwargs @@ -260,7 +259,7 @@ def get_readonly_fields(self, request, obj=None): return fields -class PartnershipAdmin(ExportMixin, CountryUsersAdminMixin, VersionAdmin): +class PartnershipAdmin(ExportMixin, CountryUsersAdminMixin, HiddenPartnerMixin, VersionAdmin): form = PartnershipForm resource_class = PCAResource # Add custom exports @@ -483,8 +482,7 @@ class GovernmentInterventionAdmin(admin.ModelAdmin): def formfield_for_foreignkey(self, db_field, request=None, **kwargs): if db_field.rel.to is PartnerOrganization: kwargs['queryset'] = PartnerOrganization.objects.filter( - partner_type=u'Government', - ) + partner_type=u'Government', hidden=False) return super(GovernmentInterventionAdmin, self).formfield_for_foreignkey( db_field, request, **kwargs @@ -713,7 +711,7 @@ class BankDetailsInlineAdmin(admin.StackedInline): extra = 1 -class AgreementAdmin(CountryUsersAdminMixin, admin.ModelAdmin): +class AgreementAdmin(HiddenPartnerMixin, CountryUsersAdminMixin, admin.ModelAdmin): form = AgreementForm list_filter = ( u'partner', @@ -789,13 +787,11 @@ class FundingCommitmentAdmin(admin.ModelAdmin): ) list_filter = ( u'grant', - u'intervention', ) list_display = ( + u'fc_ref', u'grant', - u'intervention', u'fr_number', - u'fc_ref', u'fr_item_amount_usd', u'agreement_amount', u'commitment_amount', diff --git a/EquiTrack/partners/migrations/0067_auto_20160910_0836.py b/EquiTrack/partners/migrations/0067_auto_20160910_0836.py new file mode 100644 index 0000000000..6add8c16bc --- /dev/null +++ b/EquiTrack/partners/migrations/0067_auto_20160910_0836.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('partners', '0066_auto_20160826_2026'), + ] + + operations = [ + migrations.AlterField( + model_name='fundingcommitment', + name='fc_ref', + field=models.CharField(max_length=50, unique=True, null=True, blank=True), + ), + migrations.AlterField( + model_name='fundingcommitment', + name='grant', + field=models.ForeignKey(blank=True, to='funds.Grant', null=True), + ), + ] diff --git a/EquiTrack/partners/migrations/0068_remove_fundingcommitment_intervention.py b/EquiTrack/partners/migrations/0068_remove_fundingcommitment_intervention.py new file mode 100644 index 0000000000..bae55770f5 --- /dev/null +++ b/EquiTrack/partners/migrations/0068_remove_fundingcommitment_intervention.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('partners', '0067_auto_20160910_0836'), + ] + + operations = [ + migrations.RemoveField( + model_name='fundingcommitment', + name='intervention', + ), + ] diff --git a/EquiTrack/partners/migrations/0069_auto_20160915_2222.py b/EquiTrack/partners/migrations/0069_auto_20160915_2222.py new file mode 100644 index 0000000000..e2276d1434 --- /dev/null +++ b/EquiTrack/partners/migrations/0069_auto_20160915_2222.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('partners', '0068_remove_fundingcommitment_intervention'), + ] + + operations = [ + migrations.AlterField( + model_name='governmentintervention', + name='result_structure', + field=models.ForeignKey(to='reports.ResultStructure', on_delete=django.db.models.deletion.DO_NOTHING), + ), + migrations.AlterField( + model_name='pca', + name='result_structure', + field=models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, blank=True, to='reports.ResultStructure', help_text='Which result structure does this partnership report under?', null=True), + ), + ] diff --git a/EquiTrack/partners/migrations/0070_auto_20160915_2340.py b/EquiTrack/partners/migrations/0070_auto_20160915_2340.py new file mode 100644 index 0000000000..84520627ef --- /dev/null +++ b/EquiTrack/partners/migrations/0070_auto_20160915_2340.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import smart_selects.db_fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('partners', '0069_auto_20160915_2222'), + ] + + operations = [ + migrations.AlterField( + model_name='ramindicator', + name='indicator', + field=smart_selects.db_fields.ChainedForeignKey(chained_model_field=b'result', chained_field=b'result', blank=True, auto_choose=True, to='reports.Indicator', null=True), + ), + ] diff --git a/EquiTrack/partners/mixins.py b/EquiTrack/partners/mixins.py index 563c2c7c24..0a173600a8 100644 --- a/EquiTrack/partners/mixins.py +++ b/EquiTrack/partners/mixins.py @@ -6,7 +6,7 @@ from django.contrib.admin.options import flatten_fieldsets from django.contrib.auth.models import Group -from partners.models import PCASector +from partners.models import PCASector, PartnerOrganization @@ -74,4 +74,15 @@ def get_sector(self, request): def get_pca(self, request): if not getattr(self, '_pca', False): self._pca = self.get_sector_from_request(request).pca - return self._pca \ No newline at end of file + return self._pca + + +class HiddenPartnerMixin(object): + + def formfield_for_foreignkey(self, db_field, request=None, **kwargs): + if db_field.name == u'partner': + kwargs['queryset'] = PartnerOrganization.objects.filter(hidden=False) + + return super(HiddenPartnerMixin, self).formfield_for_foreignkey( + db_field, request, **kwargs + ) \ No newline at end of file diff --git a/EquiTrack/partners/models.py b/EquiTrack/partners/models.py index cae05e4412..def4c1bb5b 100644 --- a/EquiTrack/partners/models.py +++ b/EquiTrack/partners/models.py @@ -746,7 +746,7 @@ class PCA(AdminURLMixin, models.Model): ) result_structure = models.ForeignKey( ResultStructure, - blank=True, null=True, + blank=True, null=True, on_delete=models.DO_NOTHING, help_text=u'Which result structure does this partnership report under?' ) number = models.CharField( @@ -849,7 +849,7 @@ class Meta: def __unicode__(self): return u'{}: {}'.format( self.partner.name, - self.reference_number + self.number if self.number else self.reference_number ) @property @@ -1090,7 +1090,7 @@ class GovernmentIntervention(models.Model): related_name='work_plans', ) result_structure = models.ForeignKey( - ResultStructure, + ResultStructure, on_delete=models.DO_NOTHING ) number = models.CharField( max_length=45L, @@ -1481,6 +1481,8 @@ class RAMIndicator(models.Model): chained_model_field="result", show_all=False, auto_choose=True, + blank=True, + null=True ) @property @@ -1492,8 +1494,7 @@ def target(self): return self.indicator.target def __unicode__(self): - return u'{} -> {} -> {}'.format( - self.result.result_structure.name, + return u'{} -> {}'.format( self.result.sector.name if self.result.sector else '', self.result.__unicode__(), ) @@ -1534,7 +1535,7 @@ def total(self): def __unicode__(self): return u'{} -> {} -> {}'.format( - self.result.result_structure.name, + self.result.result_structure.name if self.result.result_structure else '', self.result.sector.name if self.result.sector else '', self.result.__unicode__(), ) @@ -1640,19 +1641,24 @@ def send_distribution(cls, sender, instance, created, **kwargs): post_save.connect(DistributionPlan.send_distribution, sender=DistributionPlan) +class FCManager(models.Manager): + def get_queryset(self): + return super(FCManager, self).get_queryset().select_related('grant__donor__name') + + class FundingCommitment(TimeFramedModel): - grant = models.ForeignKey(Grant) - intervention = models.ForeignKey(PCA, null=True, related_name='funding_commitments') + grant = models.ForeignKey(Grant, null=True, blank=True) fr_number = models.CharField(max_length=50) wbs = models.CharField(max_length=50) fc_type = models.CharField(max_length=50) - fc_ref = models.CharField(max_length=50, blank=True, null=True) + fc_ref = models.CharField(max_length=50, blank=True, null=True, unique=True) fr_item_amount_usd = models.DecimalField(decimal_places=2, max_digits=12, blank=True, null=True) agreement_amount = models.DecimalField(decimal_places=2, max_digits=12, blank=True, null=True) commitment_amount = models.DecimalField(decimal_places=2, max_digits=12, blank=True, null=True) expenditure_amount = models.DecimalField(decimal_places=2, max_digits=12, blank=True, null=True) + objects = FCManager() class DirectCashTransfer(models.Model): diff --git a/EquiTrack/partners/templatetags/intervention_tags.py b/EquiTrack/partners/templatetags/intervention_tags.py index ed92f46da0..77f9de3f1e 100644 --- a/EquiTrack/partners/templatetags/intervention_tags.py +++ b/EquiTrack/partners/templatetags/intervention_tags.py @@ -108,7 +108,7 @@ def show_fr_fc(value): return '' intervention = PCA.objects.get(id=int(value)) - commitments = FundingCommitment.objects.filter(intervention=intervention) + commitments = FundingCommitment.objects.filter(fr_number=intervention.fr_number) data = tablib.Dataset() fr_fc_summary = [] diff --git a/EquiTrack/partners/tests/test_models.py b/EquiTrack/partners/tests/test_models.py index 3fa83df855..6b35fe6439 100644 --- a/EquiTrack/partners/tests/test_models.py +++ b/EquiTrack/partners/tests/test_models.py @@ -130,7 +130,6 @@ def setUp(self): start=current_cp.from_date, end=current_cp.from_date+datetime.timedelta(days=200), grant=grant, - intervention=self.intervention, fr_number='0123456789', wbs='Test', fc_type='PCA', @@ -140,7 +139,6 @@ def setUp(self): start=current_cp.from_date+datetime.timedelta(days=200), end=current_cp.to_date, grant=grant, - intervention=self.intervention, fr_number='0123456789', wbs='Test', fc_type='PCA', @@ -152,10 +150,11 @@ def test_planned_cash_transfers(self): total = self.intervention.partner.planned_cash_transfers self.assertEqual(total, 60000) - def test_actual_cash_transferred(self): - total = self.intervention.partner.actual_cash_transferred - self.assertEqual(total, 40000) + # The total cash transferred and actual cash transferred for the partner now comes from vision + # def test_actual_cash_transferred(self): + # total = self.intervention.partner.actual_cash_transferred + # self.assertEqual(total, 40000) - def test_total_cash_transferred(self): - total = self.intervention.partner.total_cash_transferred - self.assertEqual(total, 80000) \ No newline at end of file + # def test_total_cash_transferred(self): + # total = self.intervention.partner.total_cash_transferred + # self.assertEqual(total, 80000) diff --git a/EquiTrack/reports/admin.py b/EquiTrack/reports/admin.py index 7d12b2e7c8..24cd2c8025 100644 --- a/EquiTrack/reports/admin.py +++ b/EquiTrack/reports/admin.py @@ -16,6 +16,7 @@ ResultStructure, ResultType, Result, + CountryProgramme ) from .forms import IndicatorAdminForm @@ -57,12 +58,6 @@ class SectorAdmin(admin.ModelAdmin): list_editable = ('color', 'dashboard',) -class ResultStructureAdmin(admin.ModelAdmin): - search_fields = ('name',) - list_filter = ('result_structure', SectorListFilter,) - list_display = ('name', 'sector', 'result_structure',) - - class GoalAdmin(admin.ModelAdmin): form = AutoSizeTextForm @@ -111,7 +106,7 @@ class GoalAdmin(admin.ModelAdmin): class IndicatorAdmin(admin.ModelAdmin): form = IndicatorAdminForm - search_fields = ('name',) + search_fields = ('name','code') list_editable = ( 'view_on_dashboard', ) @@ -163,7 +158,7 @@ class ResultAdmin(MPTTModelAdmin): 'name', ) list_filter = ( - 'result_structure', + 'country_programme', 'sector', 'result_type', HiddenResultFilter, @@ -201,6 +196,7 @@ def show_results(self, request, queryset): admin.site.register(Result, ResultAdmin) admin.site.register(ResultStructure) +admin.site.register(CountryProgramme) admin.site.register(Sector, SectorAdmin) admin.site.register(Goal, GoalAdmin) admin.site.register(Unit, ImportExportModelAdmin) diff --git a/EquiTrack/reports/fixtures/initial_data.json b/EquiTrack/reports/fixtures/initial_data.json index bafcbc86c7..0be00b32f0 100644 --- a/EquiTrack/reports/fixtures/initial_data.json +++ b/EquiTrack/reports/fixtures/initial_data.json @@ -1,17 +1,22 @@ [ - { - "pk": 1, - "model": "reports.resulttype", - "fields": {"name": "Outcome"} - }, - { - "pk": 2, - "model": "reports.resulttype", - "fields": {"name": "Output"} - }, - { - "pk": 3, - "model": "reports.resulttype", - "fields": {"name": "Activity"} - } -] \ No newline at end of file + { + "pk": 1, + "model": "reports.resulttype", + "fields": {"name": "Outcome"} + }, + { + "pk": 2, + "model": "reports.resulttype", + "fields": {"name": "Output"} + }, + { + "pk": 3, + "model": "reports.resulttype", + "fields": {"name": "Activity"} + }, + { + "pk": 4, + "model": "reports.resulttype", + "fields": {"name": "Sub-activity"} + } +] diff --git a/EquiTrack/reports/migrations/0020_indicator_assumptions.py b/EquiTrack/reports/migrations/0020_indicator_assumptions.py new file mode 100644 index 0000000000..54d5f69a65 --- /dev/null +++ b/EquiTrack/reports/migrations/0020_indicator_assumptions.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reports', '0019_auto_20160825_1857'), + ] + + operations = [ + migrations.AddField( + model_name='indicator', + name='assumptions', + field=models.TextField(null=True, blank=True), + ), + ] diff --git a/EquiTrack/reports/migrations/0021_auto_20160906_1925.py b/EquiTrack/reports/migrations/0021_auto_20160906_1925.py new file mode 100644 index 0000000000..dfb82f76bc --- /dev/null +++ b/EquiTrack/reports/migrations/0021_auto_20160906_1925.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reports', '0020_indicator_assumptions'), + ] + + operations = [ + migrations.CreateModel( + name='CountryProgramme', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=150)), + ('wbs', models.CharField(unique=True, max_length=30)), + ('from_date', models.DateField()), + ('to_date', models.DateField()), + ], + ), + migrations.AlterField( + model_name='result', + name='result_structure', + field=models.ForeignKey(blank=True, to='reports.ResultStructure', null=True), + ), + migrations.AlterField( + model_name='resulttype', + name='name', + field=models.CharField(unique=True, max_length=150, choices=[(b'Outcome', b'Outcome'), (b'Output', b'Output'), (b'Activity', b'Activity'), (b'Sub-Activity', b'Sub-Activity')]), + ), + migrations.AlterUniqueTogether( + name='indicator', + unique_together=set([]), + ), + migrations.AddField( + model_name='result', + name='country_programme', + field=models.ForeignKey(blank=True, to='reports.CountryProgramme', null=True), + ), + migrations.AddField( + model_name='resultstructure', + name='country_programme', + field=models.ForeignKey(blank=True, to='reports.CountryProgramme', null=True), + ), + ] diff --git a/EquiTrack/reports/migrations/0022_auto_20160906_1927.py b/EquiTrack/reports/migrations/0022_auto_20160906_1927.py new file mode 100644 index 0000000000..c98f773a54 --- /dev/null +++ b/EquiTrack/reports/migrations/0022_auto_20160906_1927.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reports', '0021_auto_20160906_1925'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='indicator', + unique_together=set([('name', 'result', 'sector')]), + ), + migrations.AlterUniqueTogether( + name='result', + unique_together=set([('wbs', 'country_programme')]), + ), + migrations.AlterUniqueTogether( + name='resultstructure', + unique_together=set([('name', 'from_date', 'to_date')]), + ), + ] diff --git a/EquiTrack/reports/migrations/0023_auto_20160909_2203.py b/EquiTrack/reports/migrations/0023_auto_20160909_2203.py new file mode 100644 index 0000000000..9e61c86ffa --- /dev/null +++ b/EquiTrack/reports/migrations/0023_auto_20160909_2203.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reports', '0022_auto_20160906_1927'), + ] + + operations = [ + migrations.AlterField( + model_name='indicator', + name='activity_info_indicators', + field=models.ManyToManyField(to='activityinfo.Indicator', blank=True), + ), + ] diff --git a/EquiTrack/reports/migrations/0024_auto_20160915_2222.py b/EquiTrack/reports/migrations/0024_auto_20160915_2222.py new file mode 100644 index 0000000000..de9c579d4b --- /dev/null +++ b/EquiTrack/reports/migrations/0024_auto_20160915_2222.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('reports', '0023_auto_20160909_2203'), + ] + + operations = [ + migrations.AlterField( + model_name='goal', + name='result_structure', + field=models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, blank=True, to='reports.ResultStructure', null=True), + ), + migrations.AlterField( + model_name='indicator', + name='result_structure', + field=models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, blank=True, to='reports.ResultStructure', null=True), + ), + migrations.AlterField( + model_name='result', + name='result_structure', + field=models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, blank=True, to='reports.ResultStructure', null=True), + ), + ] diff --git a/EquiTrack/reports/models.py b/EquiTrack/reports/models.py index bdceb07ad7..d780473dae 100644 --- a/EquiTrack/reports/models.py +++ b/EquiTrack/reports/models.py @@ -1,5 +1,4 @@ -__author__ = 'jcranwellward' - +from datetime import datetime from django.db import models import django.contrib.postgres.fields as pgfields from jsonfield import JSONField @@ -15,15 +14,34 @@ from django.utils.functional import cached_property -# TODO: move to the global schema + +class CountryProgramme(models.Model): + name = models.CharField(max_length=150) + wbs = models.CharField(max_length=30, unique=True) + from_date = models.DateField() + to_date = models.DateField() + + + def __unicode__(self): + return ' '.join([self.name, self.wbs]) + + @classmethod + def current(cls): + today = datetime.now() + return cls.objects.get(to_date__gte=today, from_date__lt=today) + + class ResultStructure(models.Model): name = models.CharField(max_length=150) + country_programme = models.ForeignKey(CountryProgramme, null=True, blank=True) from_date = models.DateField() to_date = models.DateField() + # TODO: add validation these dates should never extend beyond the country programme structure class Meta: ordering = ['name'] + unique_together = (("name", "from_date", "to_date"),) def __unicode__(self): return self.name @@ -34,8 +52,18 @@ def current(cls): class ResultType(models.Model): - - name = models.CharField(max_length=150) + OUTCOME = 'Outcome' + OUTPUT = 'Output' + ACTIVITY = 'Activity' + SUBACTIVITY = 'Sub-Activity' + + NAME_CHOICES = ( + (OUTCOME, 'Outcome'), + (OUTPUT, 'Output'), + (ACTIVITY, 'Activity'), + (SUBACTIVITY, 'Sub-Activity'), + ) + name = models.CharField(max_length=150, unique=True, choices=NAME_CHOICES) def __unicode__(self): return self.name @@ -74,12 +102,13 @@ def __unicode__(self): class ResultManager(models.Manager): def get_queryset(self): - return super(ResultManager, self).get_queryset().select_related('result_structure', 'result_type') + return super(ResultManager, self).get_queryset().select_related('country_programme', 'result_structure', 'result_type') class Result(MPTTModel): - result_structure = models.ForeignKey(ResultStructure) + result_structure = models.ForeignKey(ResultStructure, null=True, blank=True, on_delete=models.DO_NOTHING) + country_programme = models.ForeignKey(CountryProgramme, null=True, blank=True) result_type = models.ForeignKey(ResultType) sector = models.ForeignKey(Sector, null=True, blank=True) name = models.TextField() @@ -126,6 +155,7 @@ class Result(MPTTModel): class Meta: ordering = ['name'] + unique_together = (('wbs', 'country_programme'),) @cached_property def result_name(self): @@ -142,8 +172,14 @@ def __unicode__(self): self.name ) - def save(self, *args, **kwargs): + def valid_entry(self): + if self.wbs: + return self.wbs.startswith(self.country_programme.wbs) + def save(self, *args, **kwargs): + # TODO add a validator that makes sure that the current result wbs fits within the countryProgramme wbs + if not self.wbs: + self.wbs = None super(Result, self).save(*args, **kwargs) nodes = self.get_descendants() for node in nodes: @@ -162,7 +198,7 @@ class Milestone(models.Model): class Goal(models.Model): result_structure = models.ForeignKey( - ResultStructure, blank=True, null=True) + ResultStructure, blank=True, null=True, on_delete=models.DO_NOTHING) sector = models.ForeignKey(Sector, related_name='goals') name = models.CharField(max_length=512L, unique=True) description = models.CharField(max_length=512L, blank=True) @@ -193,7 +229,7 @@ class Indicator(models.Model): ) result_structure = models.ForeignKey( ResultStructure, - blank=True, null=True + blank=True, null=True, on_delete=models.DO_NOTHING ) result = models.ForeignKey(Result, null=True, blank=True) @@ -205,6 +241,7 @@ class Indicator(models.Model): sector_total = models.IntegerField(verbose_name='Sector Target', null=True, blank=True) current = models.IntegerField(null=True, blank=True, default=0) sector_current = models.IntegerField(null=True, blank=True) + assumptions = models.TextField(null=True, blank=True) # RAM Info target = models.CharField(max_length=255, null=True, blank=True) @@ -216,6 +253,7 @@ class Indicator(models.Model): in_activity_info = models.BooleanField(default=False) activity_info_indicators = models.ManyToManyField( 'activityinfo.Indicator', + blank=True ) class Meta: @@ -254,3 +292,9 @@ def progress(self, result_structure=None): ) total = programmed.aggregate(models.Sum('current_progress')) return (total[total.keys()[0]] or 0) + self.current if self.current else 0 + + def save(self, *args, **kwargs): + # Prevent from saving empty strings as code because of the unique together constraint + if not self.code: + self.code = None + super(Indicator, self).save(*args, **kwargs) diff --git a/EquiTrack/reports/tests/test_views.py b/EquiTrack/reports/tests/test_views.py index 4900490043..2f722ad729 100644 --- a/EquiTrack/reports/tests/test_views.py +++ b/EquiTrack/reports/tests/test_views.py @@ -1,6 +1,7 @@ __author__ = 'achamseddine' import random +import factory from rest_framework import status @@ -28,6 +29,10 @@ def setUp(self): self.milestone1 = MilestoneFactory(result=self.result1) self.milestone2 = MilestoneFactory(result=self.result1) + # Additional data to use in tests + self.location3 = LocationFactory() + self.section3 = SectionFactory() + def test_api_resultstructures_list(self): response = self.forced_auth_req('get', '/api/reports/result-structures/', user=self.unicef_staff) @@ -50,8 +55,26 @@ def test_api_indicators_list(self): def test_api_results_list(self): response = self.forced_auth_req('get', '/api/reports/results/', user=self.unicef_staff) + self.assertEquals(response.status_code, status.HTTP_200_OK) + self.assertEquals(int(response.data[0]["id"]), self.result1.id) + def test_api_results_patch(self): + url = '/api/reports/results/{}/'.format(self.result1.id) + data = {"name": "patched name"} + response = self.forced_auth_req('patch', url, user=self.unicef_staff, data=data) + self.assertEquals(response.status_code, status.HTTP_200_OK) + self.assertEquals(response.data["name"], "patched name") + + def test_api_results_update_m2m(self): + url = '/api/reports/results/{}/'.format(self.result1.id) + data = { + "geotag": [self.location1.id, self.location3.id], + "sections": [self.section1.id, self.section3.id] + } + response = self.forced_auth_req('patch', url, user=self.unicef_staff, data=data) self.assertEquals(response.status_code, status.HTTP_200_OK) + self.assertItemsEqual(response.data["geotag"], (self.location1.id, self.location3.id,)) + self.assertItemsEqual(response.data["sections"], [self.section1.id, self.section3.id]) def test_api_units_list(self): response = self.forced_auth_req('get', '/api/reports/units/', user=self.unicef_staff) diff --git a/EquiTrack/reports/views.py b/EquiTrack/reports/views.py index cb76a4d00f..623da33b94 100644 --- a/EquiTrack/reports/views.py +++ b/EquiTrack/reports/views.py @@ -71,8 +71,7 @@ class MilestoneViewSet(viewsets.ModelViewSet): permission_classes = (IsAdminUser,) -class ResultViewSet(mixins.ListModelMixin, - viewsets.GenericViewSet): +class ResultViewSet(viewsets.ModelViewSet): """ Returns a list of all Results """ @@ -81,13 +80,13 @@ class ResultViewSet(mixins.ListModelMixin, permission_classes = (IsAdminUser,) -class IndicatorViewSet(mixins.ListModelMixin, - viewsets.GenericViewSet): +class IndicatorViewSet(viewsets.ModelViewSet): """ - Returns a list of all Indicators + CRUD api for Indicators """ queryset = Indicator.objects.all() serializer_class = IndicatorCreateSerializer + permission_classes = (IsAdminUser,) class UnitViewSet(mixins.RetrieveModelMixin, diff --git a/EquiTrack/templates/base.html b/EquiTrack/templates/base.html index 5b714139a6..cbcb7aea3d 100644 --- a/EquiTrack/templates/base.html +++ b/EquiTrack/templates/base.html @@ -157,9 +157,9 @@

You have {{ messages|length }} new messages

  • Trips
    • -
  • - HACT -
    • +{#
  • #} +{# HACT#} +{#
    • #}
  • Partners
    • diff --git a/EquiTrack/trips/admin.py b/EquiTrack/trips/admin.py index fd9cf12661..a515e99e6a 100644 --- a/EquiTrack/trips/admin.py +++ b/EquiTrack/trips/admin.py @@ -4,6 +4,7 @@ from django.contrib.auth import get_user_model from django.contrib.auth.models import Group from django.db import models, connection +from django.db.models import Q from django.forms import Textarea from reversion.admin import VersionAdmin @@ -39,6 +40,7 @@ OwnerFilter ) from reports.models import Result +from partners.models import PartnerOrganization, GovernmentIntervention from .exports import TripResource, ActionPointResource User = get_user_model() @@ -49,7 +51,14 @@ class LinkedPartnerInlineAdmin(admin.TabularInline): suit_classes = u'suit-tab suit-tab-planning' extra = 1 -from partners.models import PartnerOrganization, GovernmentIntervention + def formfield_for_foreignkey(self, db_field, request=None, **kwargs): + if db_field.name == u'partner': + kwargs['queryset'] = PartnerOrganization.objects.filter(~Q(partner_type=u'Government') & Q(hidden=False)) + + return super(LinkedPartnerInlineAdmin, self).formfield_for_foreignkey( + db_field, request, **kwargs + ) + class LinkedGovernmentPartnerInlineAdmin(admin.TabularInline): model = LinkedGovernmentPartner @@ -58,7 +67,7 @@ class LinkedGovernmentPartnerInlineAdmin(admin.TabularInline): def formfield_for_foreignkey(self, db_field, request=None, **kwargs): if db_field.name == u'partner': - kwargs['queryset'] = PartnerOrganization.objects.filter(partner_type=u'Government') + kwargs['queryset'] = PartnerOrganization.objects.filter(partner_type=u'Government', hidden=False) return super(LinkedGovernmentPartnerInlineAdmin, self).formfield_for_foreignkey( @@ -78,7 +87,7 @@ class TripFundsInlineAdmin(admin.TabularInline): formset = TripFundsForm suit_classes = u'suit-tab suit-tab-planning' extra = 1 - max_num = 1 + max_num = 10 def formfield_for_foreignkey(self, db_field, request=None, **kwargs): if db_field.name == u'result': diff --git a/EquiTrack/vision/adapters/funding.py b/EquiTrack/vision/adapters/funding.py index e93edd3e92..b4f5c897e2 100644 --- a/EquiTrack/vision/adapters/funding.py +++ b/EquiTrack/vision/adapters/funding.py @@ -2,7 +2,8 @@ from vision.vision_data_synchronizer import VisionDataSynchronizer -from vision.utils import wcf_json_date_as_datetime +from vision.utils import wcf_json_date_as_datetime, comp_decimals +from django.utils import timezone from funds.models import Grant, Donor from partners.models import FundingCommitment, DirectCashTransfer, PCA @@ -30,6 +31,16 @@ class FundingSynchronizer(VisionDataSynchronizer): "COMMITMENT_AMT", # NUMBER Commitment Amount "EXPENDITURE_AMT", # NUMBER Commitment Amount ) + MAPPING = { + 'start': "FR_START_DATE", + 'end': "FR_END_DATE", + 'wbs': "IR_WBS", + 'fc_type': "COMMITMENT_DOC_TYPE", + 'fr_item_amount_usd': "FR_ITEM_AMT", + 'agreement_amount': "AGREEMENT_AMT", + 'commitment_amount': "COMMITMENT_AMT", + 'expenditure_amount': "EXPENDITURE_AMT" + } def _convert_records(self, records): return json.loads(records) @@ -38,7 +49,8 @@ def _filter_records(self, records): records = super(FundingSynchronizer, self)._filter_records(records) def bad_record(record): - if record['GRANT_REF'] == 'Unknown': + # We don't care about FCs without expenditure + if not record['EXPENDITURE_AMT']: return False if not record['FR_DOC_NUMBER']: return False @@ -46,43 +58,85 @@ def bad_record(record): return filter(bad_record, records) + def _save_records(self, records): processed = 0 filtered_records = self._filter_records(records) + fetched_grants = {} + fcs = {} + + def _changed_fields( fields, local_obj, api_obj): + for field in fields: + apiobj_field = api_obj[self.MAPPING[field]] + if field in ['fr_item_amount_usd','agreement_amount', 'commitment_amount', 'expenditure_amount']: + return not comp_decimals(getattr(local_obj, field), apiobj_field) + + if field in ['start', 'end']: + if not wcf_json_date_as_datetime(api_obj[self.MAPPING[field]]): + apiobj_field = None + else: + apiobj_field = timezone.make_aware(wcf_json_date_as_datetime(api_obj[self.MAPPING[field]]), + timezone.get_default_timezone()) + if field == 'fc_type': + apiobj_field = api_obj[self.MAPPING[field]] or 'No Record' + if getattr(local_obj, field) != apiobj_field: + print "field changed", field + return True + return False + + for fc_line in filtered_records: - try: - grant = Grant.objects.get( - name=fc_line["GRANT_REF"], - ) - except Grant.DoesNotExist: - print 'Grant: {} does not exist'.format(fc_line["GRANT_REF"]) + grant = None + saving = False + if fc_line['GRANT_REF'] == 'Unknown': + # This is a non-grant commitment + pass else: try: - funding_commitment, created = FundingCommitment.objects.get_or_create( + grant = fetched_grants[fc_line["GRANT_REF"]] + except KeyError: + try: + grant = Grant.objects.get( + name=fc_line["GRANT_REF"], + ) + except Grant.DoesNotExist: + print 'Grant: {} does not exist'.format(fc_line["GRANT_REF"]) + continue + else: + fetched_grants[fc_line["GRANT_REF"]] = grant + + try: + fc = fcs[fc_line["COMMITMENT_REF"]] + # if there are multiple fcs in the response it means their total needs to be aggregated + fc.expenditure_amount += fc_line.get("EXPENDITURE_AMT", 0) + + except KeyError: + + try: + fc, saving = FundingCommitment.objects.get_or_create( grant=grant, fr_number=fc_line["FR_DOC_NUMBER"], fc_ref=fc_line["COMMITMENT_REF"] ) - funding_commitment.start = wcf_json_date_as_datetime(fc_line["FR_START_DATE"]) - funding_commitment.end = wcf_json_date_as_datetime(fc_line["FR_END_DATE"]) - funding_commitment.wbs = fc_line["IR_WBS"] - funding_commitment.fc_type = fc_line["COMMITMENT_DOC_TYPE"] or 'No Record' - funding_commitment.fr_item_amount_usd = fc_line["FR_ITEM_AMT"] - funding_commitment.agreement_amount = fc_line["AGREEMENT_AMT"] - funding_commitment.commitment_amount = fc_line["COMMITMENT_AMT"] - funding_commitment.expenditure_amount = fc_line["EXPENDITURE_AMT"] - try: - intervention = PCA.objects.get(fr_number=fc_line["FR_DOC_NUMBER"]) - funding_commitment.intervention = intervention - except PCA.DoesNotExist: - pass - funding_commitment.save() except FundingCommitment.MultipleObjectsReturned as exp: exp.message += 'FC Ref ' + fc_line["COMMITMENT_REF"] raise - processed += 1 + fc_fields = ['start', 'end', 'wbs', 'fc_type', 'fr_item_amount_usd', + 'agreement_amount', 'commitment_amount', 'expenditure_amount'] + if saving or _changed_fields(fc_fields, fc, fc_line): + fc.start = wcf_json_date_as_datetime(fc_line["FR_START_DATE"]) + fc.end = wcf_json_date_as_datetime(fc_line["FR_END_DATE"]) + fc.wbs = fc_line["IR_WBS"] + fc.fc_type = fc_line["COMMITMENT_DOC_TYPE"] or 'No Record' + fc.fr_item_amount_usd = fc_line["FR_ITEM_AMT"] + fc.agreement_amount = fc_line["AGREEMENT_AMT"] + fc.commitment_amount = fc_line["COMMITMENT_AMT"] + fc.expenditure_amount = fc_line["EXPENDITURE_AMT"] + fc.save() + + processed += 1 return processed @@ -132,3 +186,5 @@ def _save_records(self, records): processed += 1 return processed + + diff --git a/EquiTrack/vision/adapters/partner.py b/EquiTrack/vision/adapters/partner.py index aed3ebec75..4938247209 100644 --- a/EquiTrack/vision/adapters/partner.py +++ b/EquiTrack/vision/adapters/partner.py @@ -1,10 +1,11 @@ import json +import decimal from django.db import IntegrityError from vision.vision_data_synchronizer import VisionDataSynchronizer from django.db import transaction -from vision.utils import wcf_json_date_as_datetime +from vision.utils import wcf_json_date_as_datetime, comp_decimals from funds.models import Grant, Donor from partners.models import PartnerOrganization @@ -42,9 +43,20 @@ class PartnerSynchronizer(VisionDataSynchronizer): "TOTAL_CASH_TRANSFERRED_CY", ) - def _get_json(self, data): - return [] if data == self.NO_DATA_MESSAGE else data + MAPPING = { + 'name': "VENDOR_NAME", + 'cso_type': 'CSO_TYPE_NAME', + 'rating': 'RISK_RATING_NAME', + 'type_of_assessment': "TYPE_OF_ASSESSMENT", + 'address': "STREET_ADDRESS", + 'phone_number': 'PHONE_NUMBER', + 'email': "EMAIL", + 'deleted_flag': "DELETED_FLAG", + 'last_assessment_date': "LAST_ASSESSMENT_DATE", + 'core_values_assessment_date': "CORE_VALUE_ASSESSMENT_DT", + 'partner_type': "PARTNER_TYPE_DESC", + } def _convert_records(self, records): return json.loads(records) @@ -58,72 +70,187 @@ def bad_record(record): return filter(bad_record, records) - @transaction.atomic - def _transactional_save(self, processed, partner): - try: - # Populate grants during import - donor = Donor.objects.get_or_create(name=partner["DONOR_NAME"])[0] - try: - grant = Grant.objects.get(name=partner["GRANT_REF"]) - except Grant.DoesNotExist: - grant = Grant.objects.create(name=partner["GRANT_REF"], donor=donor) + def _get_json(self, data): + return [] if data == self.NO_DATA_MESSAGE else data + + def update_stuff(self, records): + _pos = [] + _vendors = [] + _donors = {} + _grants = {} + _totals_cy = {} + _totals_cp = {} + + + def _changed_fields( fields, local_obj, api_obj): + for field in fields: + apiobj_field = api_obj[self.MAPPING[field]] + + if field.endswith('date'): + if not wcf_json_date_as_datetime(api_obj[self.MAPPING[field]]): + apiobj_field = None + else: + apiobj_field = wcf_json_date_as_datetime(api_obj[self.MAPPING[field]]).date() + + if field == 'partner_type': + apiobj_field = type_mapping[api_obj[self.MAPPING[field]]] + + if field == 'deleted_flag': + apiobj_field = True if api_obj[self.MAPPING[field]] else False + + if getattr(local_obj, field) != apiobj_field: + print "field changed", field + return True + return False + + + + + + def _process_po(po_api): + if po_api['VENDOR_CODE'] not in _vendors: + _pos.append(po_api) + _vendors.append(po_api['VENDOR_CODE']) + + if not _donors.get(po_api["DONOR_NAME"], None): + temp_donor = Donor.objects.get_or_create(name=po_api["DONOR_NAME"])[0] + _donors[po_api["DONOR_NAME"]] = temp_donor + + donor_grant_pair = po_api["DONOR_NAME"] + po_api["GRANT_REF"] + if not _grants.get(donor_grant_pair, None): + try: + temp_grant = Grant.objects.get(name=po_api["GRANT_REF"]) + except Grant.DoesNotExist: + temp_grant = Grant.objects.create( + name=po_api["GRANT_REF"], + donor=_donors[po_api["DONOR_NAME"]] + ) + + temp_grant.description = po_api["GRANT_DESC"] + if po_api["EXPIRY_DATE"] is not None: + temp_grant.expiry = wcf_json_date_as_datetime(po_api["EXPIRY_DATE"]) + temp_grant.save() + _grants[donor_grant_pair] = temp_grant + + if not po_api["TOTAL_CASH_TRANSFERRED_CP"]: + po_api["TOTAL_CASH_TRANSFERRED_CP"] = 0 + if not po_api["TOTAL_CASH_TRANSFERRED_CY"]: + po_api["TOTAL_CASH_TRANSFERRED_CY"] = 0 + + if not _totals_cp.get(po_api['VENDOR_CODE']): + _totals_cp[po_api['VENDOR_CODE']] = po_api["TOTAL_CASH_TRANSFERRED_CP"] else: - grant.donor = donor - grant.description = partner["GRANT_DESC"] - if partner["EXPIRY_DATE"] is not None: - grant.expiry = wcf_json_date_as_datetime(partner["EXPIRY_DATE"]) - grant.save() + _totals_cp[po_api['VENDOR_CODE']] += po_api["TOTAL_CASH_TRANSFERRED_CP"] + + if not _totals_cy.get(po_api['VENDOR_CODE']): + _totals_cy[po_api['VENDOR_CODE']] = po_api["TOTAL_CASH_TRANSFERRED_CY"] + else: + _totals_cy[po_api['VENDOR_CODE']] += po_api["TOTAL_CASH_TRANSFERRED_CY"] + + + + + + + def _partner_save(processed, partner): + try: - partner_org = PartnerOrganization.objects.get(vendor_number=partner["VENDOR_CODE"]) - except PartnerOrganization.DoesNotExist: - partner_org = PartnerOrganization(vendor_number=partner["VENDOR_CODE"]) - - # partner_org, created = PartnerOrganization.objects.get_or_create( - # vendor_number=partner["VENDOR_CODE"] - # ) - partner_org.name = partner["VENDOR_NAME"] - partner_org.partner_type = type_mapping[partner["PARTNER_TYPE_DESC"]] - partner_org.cso_type = partner["CSO_TYPE_NAME"] - partner_org.rating = partner["RISK_RATING_NAME"] - partner_org.type_of_assessment = partner["TYPE_OF_ASSESSMENT"] - partner_org.last_assessment_date = wcf_json_date_as_datetime(partner["LAST_ASSESSMENT_DATE"]) - partner_org.address = partner["STREET_ADDRESS"] - partner_org.phone_number = partner["PHONE_NUMBER"] - partner_org.email = partner["EMAIL"] - partner_org.core_values_assessment_date = wcf_json_date_as_datetime(partner["CORE_VALUE_ASSESSMENT_DT"]) - partner_org.total_ct_cp = partner["TOTAL_CASH_TRANSFERRED_CP"] or 0 - partner_org.total_ct_cy = partner["TOTAL_CASH_TRANSFERRED_CY"] or 0 - partner_org.deleted_flag = True if partner["DELETED_FLAG"] else False - partner_org.hidden = partner_org.deleted_flag - - partner_org.vision_synced = True - partner_org.save() - processed += 1 - - except KeyError as exp: - print "Partner {} skipped, because PartnerType ={}".format( - partner['VENDOR_NAME'], exp - ) - # if partner organization exists in etools db (these are nameless) - if partner_org.id: - partner_org.name = ""# leaving the name blank on purpose (invalid record) - partner_org.deleted_flag = True if partner["DELETED_FLAG"] else False - partner_org.hidden = True - partner_org.save() - except Exception as exp: - print "Exception message: {} " \ - "Exception type: {} " \ - "Exception args: {} ".format( - exp.message, type(exp).__name__, exp.args - ) - return processed + new = False + saving = False + try: + partner_org = PartnerOrganization.objects.get(vendor_number=partner["VENDOR_CODE"]) + except PartnerOrganization.DoesNotExist: + partner_org = PartnerOrganization(vendor_number=partner["VENDOR_CODE"]) + new = True + + try: + type_mapping[partner["PARTNER_TYPE_DESC"]] + except KeyError as exp: + print "Partner {} skipped, because PartnerType ={}".format( + partner['VENDOR_NAME'], exp + ) + # if partner organization exists in etools db (these are nameless) + if partner_org.id: + partner_org.name = ""# leaving the name blank on purpose (invalid record) + partner_org.deleted_flag = True if partner["DELETED_FLAG"] else False + partner_org.hidden = True + partner_org.save() + return processed + + + if new or _changed_fields(['name', 'cso_type', 'rating', 'type_of_assessment', + 'address', 'phone_number', 'email', 'deleted_flag', + 'last_assessment_date', 'core_values_assessment_date'], + partner_org, partner): + partner_org.name = partner["VENDOR_NAME"] + partner_org.cso_type = partner["CSO_TYPE_NAME"] + partner_org.rating = partner["RISK_RATING_NAME"] + partner_org.type_of_assessment = partner["TYPE_OF_ASSESSMENT"] + partner_org.address = partner["STREET_ADDRESS"] + partner_org.phone_number = partner["PHONE_NUMBER"] + partner_org.email = partner["EMAIL"] + partner_org.core_values_assessment_date = wcf_json_date_as_datetime(partner["CORE_VALUE_ASSESSMENT_DT"]) + partner_org.last_assessment_date = wcf_json_date_as_datetime(partner["LAST_ASSESSMENT_DATE"]) + partner_org.partner_type = type_mapping[partner["PARTNER_TYPE_DESC"]] + partner_org.deleted_flag = True if partner["DELETED_FLAG"] else False + partner_org.hidden = partner_org.deleted_flag + partner_org.vision_synced = True + saving = True + + if partner_org.total_ct_cp == None or partner_org.total_ct_cy == None or \ + not comp_decimals(partner_org.total_ct_cp, _totals_cp[partner["VENDOR_CODE"]]) or \ + not comp_decimals(partner_org.total_ct_cy, _totals_cy[partner["VENDOR_CODE"]]): + + partner_org.total_ct_cy = _totals_cy[partner["VENDOR_CODE"]] + partner_org.total_ct_cp = _totals_cp[partner["VENDOR_CODE"]] + + + + saving = True + print "sums changed", partner_org + + if saving: + print "Updating Partner", partner_org + partner_org.save() + del _totals_cy[partner["VENDOR_CODE"]] + del _totals_cp[partner["VENDOR_CODE"]] + + processed += 1 + + except Exception as exp: + print "Exception message: {} " \ + "Exception type: {} " \ + "Exception args: {} ".format( + exp.message, type(exp).__name__, exp.args + ) + return processed - def _save_records(self, records): processed = 0 filtered_records = self._filter_records(records) + for partner in filtered_records: - processed = self._transactional_save(processed, partner) + _process_po(partner) + + + for partner in _pos: + processed = _partner_save(processed, partner) + + self._pos = [] + self._vendors = [] + self._donors = {} + self._grants = {} + self._totals_cy = {} + self._totals_cp = {} + return processed + + + def _save_records(self, records): + + + + processed = self.update_stuff(records) + return processed diff --git a/EquiTrack/vision/adapters/programme.py b/EquiTrack/vision/adapters/programme.py index 716124d384..e47384f7f4 100644 --- a/EquiTrack/vision/adapters/programme.py +++ b/EquiTrack/vision/adapters/programme.py @@ -1,16 +1,16 @@ import json import datetime -from reports.models import ResultStructure, ResultType, Result, Indicator +from reports.models import ResultStructure, ResultType, Result, Indicator, CountryProgramme from vision.utils import wcf_json_date_as_datetime from vision.vision_data_synchronizer import VisionDataSynchronizer class ProgrammeSynchronizer(VisionDataSynchronizer): - ENDPOINT = 'GetProgrammeStructureList_JSON' REQUIRED_KEYS = ( "COUNTRY_PROGRAMME_NAME", + "COUNTRY_PROGRAMME_WBS", "CP_START_DATE", "CP_END_DATE", "OUTCOME_AREA_CODE", @@ -52,7 +52,7 @@ def _convert_records(self, records): def _filter_records(self, records): records = super(ProgrammeSynchronizer, self)._filter_records(records) today = datetime.datetime.today() - last_year = datetime.datetime(today.year-1, 1, 1) + last_year = datetime.datetime(today.year - 1, 1, 1) def in_time_range(record): end = wcf_json_date_as_datetime(record['OUTCOME_END_DATE']) @@ -62,72 +62,198 @@ def in_time_range(record): return filter(in_time_range, records) + def _changed_fields(self, obj_type, fields, local_obj, api_obj): + tmap = { + 'cp': { + 'name': "COUNTRY_PROGRAMME_NAME", + 'wbs': "COUNTRY_PROGRAMME_WBS", + 'from_date': 'CP_START_DATE', + 'to_date': 'CP_END_DATE' + }, + 'outcome': { + 'name': "OUTCOME_DESCRIPTION", + 'wbs': "OUTCOME_WBS", + 'from_date': 'OUTCOME_START_DATE', + 'to_date': 'OUTCOME_END_DATE' + }, + 'output': { + 'name': "OUTPUT_DESCRIPTION", + 'wbs': "OUTPUT_WBS", + 'from_date': 'OUTPUT_START_DATE', + 'to_date': 'OUTPUT_END_DATE' + }, + 'activity': { + 'name': "ACTIVITY_DESCRIPTION", + 'wbs': "ACTIVITY_WBS", + 'from_date': 'ACTIVITY_START_DATE', + 'to_date': 'ACTIVITY_END_DATE', + 'sic_code': 'SIC_CODE', + 'sic_name': 'SIC_NAME', + 'gic_code': 'GIC_CODE', + 'gic_name': 'GIC_NAME', + 'activity_focus_code': 'ACTIVITY_FOCUS_CODE', + 'activity_focus_name': 'ACTIVITY_FOCUS_NAME', + } + } + mapping = tmap[obj_type] + for field in fields: + apiobj_field = api_obj[mapping[field]] + if field.endswith('date'): + if not wcf_json_date_as_datetime(api_obj[mapping[field]]): + apiobj_field = None + else: + apiobj_field = wcf_json_date_as_datetime(api_obj[mapping[field]]).date() + if getattr(local_obj, field) != apiobj_field: + print "field changed", field, obj_type, getattr(local_obj, field), 'Changed To', apiobj_field + return True + return False + + def _update_record(self): + pass + def _save_records(self, records): processed = 0 filtered_records = self._filter_records(records) + cps = {} + outcomes = {} + outputs = {} + activities = {} + for result in filtered_records: - try: - result_structure, created = ResultStructure.objects.get_or_create( - name=result['COUNTRY_PROGRAMME_NAME'], - from_date=wcf_json_date_as_datetime(result['CP_START_DATE']), - to_date=wcf_json_date_as_datetime(result['CP_END_DATE']), - ) - except ResultStructure.MultipleObjectsReturned as exp: - exp.message += 'Result Structure: ' + result['COUNTRY_PROGRAMME_NAME'] - raise + # Find the country programme: + updating_cp = False + country_programme = cps.get(result["COUNTRY_PROGRAMME_WBS"], None) + if not country_programme: + try: + country_programme = CountryProgramme.objects.get( + wbs=result["COUNTRY_PROGRAMME_WBS"], + ) + except CountryProgramme.DoesNotExist: + country_programme = CountryProgramme( + wbs=result["COUNTRY_PROGRAMME_WBS"] + ) + updating_cp = True + except CountryProgramme.MultipleObjectsReturned as exp: + exp.message += 'Result Structure: ' + result['COUNTRY_PROGRAMME_NAME'] + raise + else: + cps[result["COUNTRY_PROGRAMME_WBS"]] = country_programme - try: - outcome, created = Result.objects.get_or_create( - result_structure=result_structure, - result_type=ResultType.objects.get_or_create(name='Outcome')[0], - wbs=result['OUTCOME_WBS'], - ) + possible_changes = ['name', 'from_date', 'to_date'] + if updating_cp or self._changed_fields('cp', possible_changes, country_programme, result): + country_programme.name = result['COUNTRY_PROGRAMME_NAME'] + country_programme.from_date = wcf_json_date_as_datetime(result['CP_START_DATE']) + country_programme.to_date = wcf_json_date_as_datetime(result['CP_END_DATE']) + print country_programme + print result + country_programme.save() + + updating_outcome = False + outcome = outcomes.get(result['OUTCOME_WBS'], None) + if not outcome: + try: + outcome, updating_outcome = Result.objects.get_or_create( + country_programme=country_programme, + result_type=ResultType.objects.get_or_create(name='Outcome')[0], + wbs=result['OUTCOME_WBS'], + ) + except Result.MultipleObjectsReturned as exp: + exp.message += 'Outcome WBS: ' + result['OUTCOME_WBS'] \ + + ' Output WBS: ' + result['OUTPUT_WBS'] \ + + ' Activity WBS: ' + result['ACTIVITY_WBS'] + raise + else: + outcomes[result['OUTCOME_WBS']] = outcome + + possible_changes = ['name', 'from_date', 'to_date'] + if updating_outcome or self._changed_fields('outcome', possible_changes, outcome, result): + # check if any of the information on the result is changing... outcome.name = result['OUTCOME_DESCRIPTION'] outcome.from_date = wcf_json_date_as_datetime(result['OUTCOME_START_DATE']) outcome.to_date = wcf_json_date_as_datetime(result['OUTCOME_END_DATE']) + if not outcome.valid_entry(): + print 'Skipping outcome because of wbs missmatch: ', outcome + # we need to skip this record since the wbs's don;t match + # TODO in these cases... send an email with the record and make the country aware + continue + # raise Exception('Wbs of outcome does not map under country_programme') outcome.save() - output, created = Result.objects.get_or_create( - result_structure=result_structure, - result_type=ResultType.objects.get_or_create(name='Output')[0], - wbs=result['OUTPUT_WBS'], - ) + updating_output = False + output = outputs.get(result['OUTPUT_WBS'], None) + if not output: + try: + output, updating_output = Result.objects.get_or_create( + country_programme=country_programme, + result_type=ResultType.objects.get_or_create(name='Output')[0], + wbs=result['OUTPUT_WBS'], + ) + except Result.MultipleObjectsReturned as exp: + exp.message += 'Outcome WBS: ' + result['OUTCOME_WBS'] \ + + ' Output WBS: ' + result['OUTPUT_WBS'] \ + + ' Activity WBS: ' + result['ACTIVITY_WBS'] + raise + else: + outputs[result['OUTPUT_WBS']] = output + + possible_changes = ['name', 'from_date', 'to_date'] + if updating_output or self._changed_fields('output', possible_changes, output, result): output.name = result['OUTPUT_DESCRIPTION'] output.from_date = wcf_json_date_as_datetime(result['OUTPUT_START_DATE']) output.to_date = wcf_json_date_as_datetime(result['OUTPUT_END_DATE']) output.parent = outcome + if not output.valid_entry(): + # we need to skip this record since the wbs's don;t match + # TODO in these cases... send an email with the record and make the country aware + continue + #raise Exception('Wbs of output does not map under country_programme') output.save() - activity, created = Result.objects.get_or_create( - result_structure=result_structure, + try: + activity, updating_activity = Result.objects.get_or_create( + country_programme=country_programme, result_type=ResultType.objects.get_or_create(name='Activity')[0], wbs=result['ACTIVITY_WBS'], ) + except Result.MultipleObjectsReturned as exp: + exp.message += 'Outcome WBS: ' + result['OUTCOME_WBS'] \ + + ' Output WBS: ' + result['OUTPUT_WBS'] \ + + ' Activity WBS: ' + result['ACTIVITY_WBS'] + raise + + possible_changes = ['name', 'from_date', 'to_date', 'sic_code', 'sic_name', 'gic_code', + 'gic_name', 'activity_focus_code', 'activity_focus_name'] + + if updating_activity or self._changed_fields('activity', possible_changes, activity, result): activity.name = result['ACTIVITY_DESCRIPTION'] activity.from_date = wcf_json_date_as_datetime(result['ACTIVITY_START_DATE']) activity.to_date = wcf_json_date_as_datetime(result['ACTIVITY_END_DATE']) activity.parent = output - activity.sic_code = result['SIC_CODE'] activity.sic_name = result['SIC_NAME'] activity.gic_code = result['GIC_CODE'] activity.gic_name = result['GIC_NAME'] activity.activity_focus_code = result['ACTIVITY_FOCUS_CODE'] activity.activity_focus_name = result['ACTIVITY_FOCUS_NAME'] + if not activity.valid_entry(): + activity.delete() + raise Exception('Wbs of activity does not map under country_programme') activity.save() - processed += 1 - except Result.MultipleObjectsReturned as exp: - exp.message += 'Outcome WBS: ' + result['OUTCOME_WBS'] \ - + ' Output WBS: ' + result['OUTPUT_WBS'] \ - + ' Activity WBS: ' + result['ACTIVITY_WBS'] - raise + if updating_cp: + print 'add cp', country_programme + if updating_activity: + print 'added activity', activity + if updating_outcome: + print 'add outcome', outcome + if updating_output: + print 'add output', output + processed += 1 return processed class RAMSynchronizer(VisionDataSynchronizer): - ENDPOINT = 'GetRAMInfo_JSON' REQUIRED_KEYS = ( "INDICATOR_DESCRIPTION", @@ -136,17 +262,32 @@ class RAMSynchronizer(VisionDataSynchronizer): "BASELINE", "TARGET", ) + MAPPING = { + 'name': 'INDICATOR_DESCRIPTION', + 'baseline': 'BASELINE', + 'target': 'TARGET', + 'code': 'INDICATOR_CODE', + } def _convert_records(self, records): return json.loads(records) + def _changed_fields(self, fields, local_obj, api_obj): + for field in fields: + obj_value = api_obj[self.MAPPING[field]][:255] + if field in ['name']: + obj_value = api_obj[self.MAPPING[field]][:1024] + if getattr(local_obj, field) != obj_value: + return True + return False + def _save_records(self, records): results = Result.objects.filter(result_type__name='Output') lookup = {} for result in results: if result.wbs: - lookup[result.wbs.replace('/', '')+'000'] = result + lookup[result.wbs.replace('/', '') + '000'] = result processed = 0 filtered_records = self._filter_records(records) @@ -161,13 +302,15 @@ def _save_records(self, records): result=result, ram_indicator=True, ) - indicator.name = ram_indicator['INDICATOR_DESCRIPTION'] - indicator.baseline = ram_indicator['BASELINE'] - indicator.target = ram_indicator['TARGET'] - indicator.save() + if created or self._changed_fields(['name', 'baseline', 'target'], indicator, ram_indicator): + indicator.name = ram_indicator['INDICATOR_DESCRIPTION'][:1024] + indicator.baseline = ram_indicator['BASELINE'][:255] + indicator.target = ram_indicator['TARGET'][:255] + indicator.save() - result.ram = True - result.save() + if not result.ram: + result.ram = True + result.save() processed += 1 return processed diff --git a/EquiTrack/vision/utils.py b/EquiTrack/vision/utils.py index cd0fd115ec..42608d35d7 100644 --- a/EquiTrack/vision/utils.py +++ b/EquiTrack/vision/utils.py @@ -16,4 +16,10 @@ def wcf_json_date_as_datetime(jd): if sign == '-': mm = -mm millisecs += (hh * 60 + mm) * 60000 return datetime.datetime(1970, 1, 1) \ - + datetime.timedelta(microseconds=millisecs * 1000) \ No newline at end of file + + datetime.timedelta(microseconds=millisecs * 1000) + +def comp_decimals(y, x): + def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) + + return isclose(float(x), float(y))