From 46300a3dcfbd970447888037394256b932009a43 Mon Sep 17 00:00:00 2001 From: Robert Calef Date: Sun, 5 May 2024 18:09:44 -0400 Subject: [PATCH] Recompute salary data with consistent set of neighborhoods --- .../extract_employment_opp_info.py | 120 ++---------------- .../parsed_employment_data.txt | 66 +++++----- data/transportation/mbta/zips_to_hoods.tsv | 40 ++++++ src/routes/dashboard/Dashboard.svelte | 11 +- 4 files changed, 90 insertions(+), 147 deletions(-) create mode 100644 data/transportation/mbta/zips_to_hoods.tsv diff --git a/data/employment_opportunities/extract_employment_opp_info.py b/data/employment_opportunities/extract_employment_opp_info.py index 9a4ebf0..37f901f 100755 --- a/data/employment_opportunities/extract_employment_opp_info.py +++ b/data/employment_opportunities/extract_employment_opp_info.py @@ -1,74 +1,13 @@ #!/usr/bin/env python3 import numpy as np +import pandas as pd -neighborhood_names = ["Allston/Brighton", "Back Bay/Beacon Hill", "Central Boston", "Charlestown", "Dorchester", "East Boston", "Fenway/Kenmore", "Hyde Park", "Jamaica Plain", "Mattapan", "Roslindale", "Roxbury", "South Boston", "South End", "West Roxbury", "The Port", "Neighborhood Nine", "Wellington-Harrington", "Mid-Cambridge", "North Cambridge", "Cambridge Highlands", "Strawberry Hill", "West Cambridge", "Riverside", "Cambridgeport", "Area 2/MIT", "East Cambridge", "Baldwin"] - -zip_neighborhoods = { - "02134" : "Allston/Brighton", - "02135" : "Allston/Brighton", - "02163" : "Allston/Brighton", - "02108" : "Back Bay/Beacon Hill", - "02116" : "Back Bay/Beacon Hill", - "02117" : "Back Bay/Beacon Hill", - "02123" : "Back Bay/Beacon Hill", - "02133" : "Back Bay/Beacon Hill", - "02199" : "Back Bay/Beacon Hill", - "02216" : "Back Bay/Beacon Hill", - "02217" : "Back Bay/Beacon Hill", - "02295" : "Back Bay/Beacon Hill", - "02101" : "Central Boston", - "02102" : "Central Boston", - "02103" : "Central Boston", - "02104" : "Central Boston", - "02105" : "Central Boston", - "02106" : "Central Boston", - "02107" : "Central Boston", - "02109" : "Central Boston", - "02110" : "Central Boston", - "02111" : "Central Boston", - "02112" : "Central Boston", - "02113" : "Central Boston", - "02114" : "Central Boston", - "02196" : "Central Boston", - "02201" : "Central Boston", - "02202" : "Central Boston", - "02203" : "Central Boston", - "02204" : "Central Boston", - "02205" : "Central Boston", - "02206" : "Central Boston", - "02207" : "Central Boston", - "02208" : "Central Boston", - "02209" : "Central Boston", - "02211" : "Central Boston", - "02212" : "Central Boston", - "02222" : "Central Boston", - "02293" : "Central Boston", - "02129" : "Charlestown", - "02122" : "Dorchester", - "02124" : "Dorchester", - "02125" : "Dorchester", - "02128" : "East Boston", - "02228" : "East Boston", - "02115" : "Fenway/Kenmore", - "02215" : "Fenway/Kenmore", - "02136" : "Hyde Park", - "02130" : "Jamaica Plain", - "02126" : "Mattapan", - "02131" : "Roslindale", - "02119" : "Roxbury", - "02120" : "Roxbury", - "02121" : "Roxbury", - "02127" : "South Boston", - "02210" : "South Boston", - "02118" : "South End", - "02132" : "West Roxbury", - "02139" : ["The Port", "Mid-Cambridge", "Riverside", "Cambridgeport", "Area 2/MIT"], - "02140" : "Neighborhood Nine", - "02141" : ["Wellington-Harrington", "North Cambridge", "East Cambridge"], - "02138" : ["Cambridge Highlands", "Strawberry Hill", "West Cambridge", "Baldwin"] -} - -overlapping_neighs = [["The Port", "Mid-Cambridge", "Riverside", "Cambridgeport", "Area 2/MIT"], ["Wellington-Harrington", "North Cambridge", "East Cambridge"], ["Cambridge Highlands", "Strawberry Hill", "West Cambridge", "Baldwin"]] +zip_map = pd.read_table("../transportation/mbta/zips_to_hoods.tsv") +zip_neighborhoods = {f"0{x.zip}":x.neighborhoods.split(",") for x in zip_map.itertuples()} +neighborhood_names = [] +for hoods in zip_neighborhoods.values(): + neighborhood_names.extend(hoods) +neighborhood_names = np.unique(neighborhood_names).tolist() num_employees = np.zeros(len(neighborhood_names)) num_establishments = np.zeros(len(neighborhood_names)) @@ -130,50 +69,9 @@ orig_file.close() -## calculate population sizes to normalize data - -population = np.zeros(len(neighborhood_names)) -census_file = open("DECENNIALDHC2020.P1-Data.txt", 'r') - -census_line = census_file.readline() -census_line = census_file.readline() -census_line = census_file.readline() - -while(census_line): - census_split = census_line.split() - - try: - neigh_idx = neighborhood_names.index(zip_neighborhoods[census_split[2]]) - except: - # not one of our zipcodes - census_line = census_file.readline() - continue - - population[neigh_idx] += int(census_split[3]) - - census_line = census_file.readline() - -census_file.close() - # save results to file out = open("parsed_employment_data.txt", 'w') -out.write("NEIGHBORHOOD\tNUM_EST\tNUM_EMP\tPAY_ANN\tPAY_QTR1\tPOP\tNORM_EST\tNORM_EMP\tNORM_PAY_ANN\tNORM_PAY_QTR1\tANN_PAY_DIV_EMP\tOVERLAPPING_NEIGHBORHOODS\n") +out.write("NEIGHBORHOOD\tNUM_EST\tNUM_EMP\tPAY_ANN\tPAY_QTR1\tANN_PAY_DIV_EMP\n") for i in range(len(neighborhood_names)): - out.write(neighborhood_names[i] + '\t' + str(int(num_establishments[i])) + '\t' + str(int(num_employees[i])) + '\t' + str(int(ann_payroll[i])) + '\t' + str(int(q1_payroll[i])) + '\t' + str(int(population[i])) + '\t' + str(num_establishments[i]/population[i]) + '\t' + str(num_employees[i]/population[i]) + '\t' + str(ann_payroll[i]/population[i]) + '\t' + str(q1_payroll[i]/population[i]) + '\t' + str(ann_payroll[i]/num_employees[i]) + '\t') - - # write the list of neighborhoods that overlap by zipcode, if applicable (sometimes happens in the Cambridge area) - in_list = False - to_write = "" - for overlap_list in overlapping_neighs: - if neighborhood_names[i] in overlap_list: - for j in range(len(overlap_list)): - if overlap_list[j] != neighborhood_names[i]: - to_write += overlap_list[j] + ", " - in_list = True - - if in_list: - out.write(to_write[:-2] + '\n') - else: - out.write("NA\n") - + out.write(neighborhood_names[i] + '\t' + str(int(num_establishments[i])) + '\t' + str(int(num_employees[i])) + '\t' + str(int(ann_payroll[i])) + '\t' + str(int(q1_payroll[i])) + '\t' + str(ann_payroll[i]/num_employees[i]) + '\n') out.close() diff --git a/data/employment_opportunities/parsed_employment_data.txt b/data/employment_opportunities/parsed_employment_data.txt index e3aaa7d..918b3c3 100644 --- a/data/employment_opportunities/parsed_employment_data.txt +++ b/data/employment_opportunities/parsed_employment_data.txt @@ -1,29 +1,37 @@ -NEIGHBORHOOD NUM_EST NUM_EMP PAY_ANN PAY_QTR1 POP NORM_EST NORM_EMP NORM_PAY_ANN NORM_PAY_QTR1 ANN_PAY_DIV_EMP OVERLAPPING_NEIGHBORHOODS -Allston/Brighton 1230 21554 1543275 364162 70330 0.01748898052040381 0.30646950092421443 21.943338546850562 5.177904166074222 71.60039899786582 NA -Back Bay/Beacon Hill 3377 106809 15947996 4634691 29266 0.11538987220665618 3.649593384815144 544.9325497163944 158.36434770723707 149.3132226685017 NA -Central Boston 5193 167140 25844167 7318903 38163 0.13607420800251552 4.37963472473338 677.2048057018578 191.78007494169745 154.62586454469306 NA -Charlestown 503 12664 999317 230729 19120 0.02630753138075314 0.6623430962343096 52.26553347280335 12.067416317991631 78.91006001263423 NA -Dorchester 1648 23776 1419043 334213 110283 0.014943372958660899 0.21559079821912716 12.867286889185097 3.03050334140348 59.683840847913864 NA -East Boston 853 20398 1217494 265088 43066 0.019806808154925 0.4736451028653694 28.270422142757628 6.155389402312729 59.68693009118541 NA -Fenway/Kenmore 1254 112831 8997755 2052613 58530 0.02142491030240902 1.9277464548094994 153.72894242268922 35.06941739279002 79.74541571022148 NA -Hyde Park 458 4965 275167 60201 35102 0.013047689590336732 0.1414449319127115 7.8390689989174405 1.7150304825935845 55.42134944612286 NA -Jamaica Plain 703 13852 965731 226731 38284 0.018362762511754258 0.36182217114199144 25.225446661790826 5.922343537770348 69.7178024833959 NA -Mattapan 242 1899 81688 18476 22734 0.0106448491246591 0.08353127474267617 3.593208410310548 0.8127034397818246 43.01632438125329 NA -Roslindale 455 5372 250446 55707 30544 0.014896542692509167 0.17587742273441592 8.199515453116815 1.8238279203771608 46.62062546537602 NA -Roxbury 865 13006 886976 194998 75519 0.011454071160899906 0.172221560137184 11.745070776890584 2.582105165587468 68.19744733200062 NA -South Boston 1798 68890 10619740 2950757 43747 0.04109996114019247 1.5747365533636593 242.75356024413102 67.4504994628203 154.15502975758454 NA -South End 894 20560 1402286 302364 29073 0.030750180579919514 0.7071853609878581 48.23327485983559 10.400165101640697 68.2045719844358 NA -West Roxbury 493 5323 248980 57943 27111 0.01818450075615064 0.1963409686105271 9.183726162812142 2.1372505625023055 46.774375352244974 NA -The Port 1173 52093 9139616 2335424 0 inf inf inf inf 175.4480640393143 Mid-Cambridge, Riverside, Cambridgeport, Area 2/MIT -Neighborhood Nine 503 8205 982498 254600 21447 0.023453163612626473 0.3825709889495034 45.81050962838626 11.8711241665501 119.74381474710542 NA -Wellington-Harrington 389 14403 1872232 543695 0 inf inf inf inf 129.98903006318127 North Cambridge, East Cambridge -Mid-Cambridge 1173 52093 9139616 2335424 0 inf inf inf inf 175.4480640393143 The Port, Riverside, Cambridgeport, Area 2/MIT -North Cambridge 389 14403 1872232 543695 0 inf inf inf inf 129.98903006318127 Wellington-Harrington, East Cambridge -Cambridge Highlands 1108 44379 3420910 807398 0 inf inf inf inf 77.08398116226142 Strawberry Hill, West Cambridge, Baldwin -Strawberry Hill 1108 44379 3420910 807398 0 inf inf inf inf 77.08398116226142 Cambridge Highlands, West Cambridge, Baldwin -West Cambridge 1108 44379 3420910 807398 0 inf inf inf inf 77.08398116226142 Cambridge Highlands, Strawberry Hill, Baldwin -Riverside 1173 52093 9139616 2335424 0 inf inf inf inf 175.4480640393143 The Port, Mid-Cambridge, Cambridgeport, Area 2/MIT -Cambridgeport 1173 52093 9139616 2335424 0 inf inf inf inf 175.4480640393143 The Port, Mid-Cambridge, Riverside, Area 2/MIT -Area 2/MIT 1173 52093 9139616 2335424 0 inf inf inf inf 175.4480640393143 The Port, Mid-Cambridge, Riverside, Cambridgeport -East Cambridge 389 14403 1872232 543695 0 inf inf inf inf 129.98903006318127 Wellington-Harrington, North Cambridge -Baldwin 1108 44379 3420910 807398 0 inf inf inf inf 77.08398116226142 Cambridge Highlands, Strawberry Hill, West Cambridge +NEIGHBORHOOD NUM_EST NUM_EMP PAY_ANN PAY_QTR1 ANN_PAY_DIV_EMP +Allston 539 8304 596490 139035 71.83164739884393 +Area 2/MIT 1173 52093 9139616 2335424 175.4480640393143 +Back Bay 1979 67148 9874471 2775575 147.05532554953237 +Baldwin 1108 44379 3420910 807398 77.08398116226142 +Beacon Hill 1087 19066 2139545 594518 112.21782230147907 +Brighton 656 12837 915721 217918 71.33450183064579 +Cambridge Highlands 1108 44379 3420910 807398 77.08398116226142 +Cambridgeport 1173 52093 9139616 2335424 175.4480640393143 +Charlestown 503 12664 999317 230729 78.91006001263423 +Chinatown 963 33148 4271232 1225276 128.85338481959695 +Dorchester 1648 23776 1419043 334213 59.683840847913864 +Downtown 3556 107148 20238471 6044190 188.8833296001792 +East Boston 853 20398 1217494 265088 59.68693009118541 +East Cambridge 389 14403 1872232 543695 129.98903006318127 +Fenway 540 51174 4156938 957585 81.23144565599719 +Hyde Park 458 4965 275167 60201 55.42134944612286 +Jamaica Plain 703 13852 965731 226731 69.7178024833959 +Longwood 714 61657 4840817 1095028 78.51204242827254 +Mattapan 242 1899 81688 18476 43.01632438125329 +Mid-Cambridge 1173 52093 9139616 2335424 175.4480640393143 +Mission Hill 176 3811 218842 50239 57.423773287850956 +Neighborhood Nine 503 8205 982498 254600 119.74381474710542 +North Cambridge 503 8205 982498 254600 119.74381474710542 +North End 180 1805 90787 15939 50.297506925207756 +Riverside 1173 52093 9139616 2335424 175.4480640393143 +Roslindale 455 5372 250446 55707 46.62062546537602 +Roxbury 689 9195 668134 144759 72.66275149537792 +South Boston 700 9465 770687 199845 81.42493396724775 +South Boston Waterfront 1098 59425 9849053 2750912 165.73921750105174 +South End 894 20560 1402286 302364 68.2045719844358 +Strawberry Hill 1108 44379 3420910 807398 77.08398116226142 +The Port 1173 52093 9139616 2335424 175.4480640393143 +Wellington-Harrington 389 14403 1872232 543695 129.98903006318127 +West Cambridge 1108 44379 3420910 807398 77.08398116226142 +West End 744 43436 4759674 1098352 109.57901280044203 +West Roxbury 493 5323 248980 57943 46.774375352244974 diff --git a/data/transportation/mbta/zips_to_hoods.tsv b/data/transportation/mbta/zips_to_hoods.tsv new file mode 100644 index 0000000..6f06079 --- /dev/null +++ b/data/transportation/mbta/zips_to_hoods.tsv @@ -0,0 +1,40 @@ +zip neighborhoods +2139 Mid-Cambridge,Riverside,The Port,Cambridgeport,Area 2/MIT +2138 Strawberry Hill,West Cambridge,Baldwin,Cambridge Highlands +2140 North Cambridge,Neighborhood Nine +2141 Wellington-Harrington,East Cambridge +2134 Allston +2116 Back Bay +2114 West End +2108 Beacon Hill +2135 Brighton +2113 North End +2109 Downtown +2110 Downtown +2199 Downtown +2203 Downtown +2215 Fenway +2222 Downtown +2283 Downtown +2284 Downtown +2293 Downtown +2297 Downtown +2298 Downtown +2129 Charlestown +2111 Chinatown +2121 Roxbury +2122 Dorchester +2124 Dorchester +2125 Dorchester +2128 East Boston +2136 Hyde Park +2130 Jamaica Plain +2115 Longwood +2126 Mattapan +2120 Mission Hill +2119 Roxbury +2127 South Boston +2210 South Boston Waterfront +2118 South End +2131 Roslindale +2132 West Roxbury diff --git a/src/routes/dashboard/Dashboard.svelte b/src/routes/dashboard/Dashboard.svelte index 0faeff5..32e0304 100644 --- a/src/routes/dashboard/Dashboard.svelte +++ b/src/routes/dashboard/Dashboard.svelte @@ -427,8 +427,8 @@ } }); - const bostonUrl = 'https://raw.githubusercontent.com/yoakiyama/zoning-dashboard-fp/main/data/employment_opportunities/parsed_employment_data.txt'; - salaryData = await d3.tsv(bostonUrl, (d) => { + const salaryUrl = 'https://raw.githubusercontent.com/yoakiyama/zoning-dashboard-fp/main/data/employment_opportunities/parsed_employment_data.txt'; + salaryData = await d3.tsv(salaryUrl, (d) => { return { neighborhood: d.NEIGHBORHOOD, avg_salary: +d.ANN_PAY_DIV_EMP, @@ -645,11 +645,8 @@ // Coloring of neighborhoods by salary $: { if (map && salaryLayerId && salaryColor) { - - minSalary = 40; maxSalary = 180; - map.setPaintProperty(salaryLayerId, 'fill-color', [ 'case', @@ -745,8 +742,8 @@ $ :{ if (clickedNeighborhood !== null || workingNeighborhood !== null) { const layers = [ - { id: rentOutlineLayerId}, - { id: commuteLineLayerId}, + { id: rentOutlineLayerId}, + { id: commuteLineLayerId}, { id: salaryLineLayerId}, ]; for (const layer of layers) {