Merge pull request #3490 from ProjectSidewalk/develop

v7.18.2

Dockerfile

@@ -1,35 +1,33 @@
-FROM openjdk:8-jdk-buster
+FROM eclipse-temurin:8u402-b06-jdk-focal
 
 RUN apt-get update && apt-get upgrade -y
 
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
+RUN curl -sL https://deb.nodesource.com/setup_20.x | bash -
 
-# Workaround because of bug in sbt from Debian.
-# See https://github.com/sbt/sbt/issues/6614
+# Workaround because of bug in sbt from Debian. See https://github.com/sbt/sbt/issues/6614.
 RUN wget https://scala.jfrog.io/artifactory/debian/sbt-1.8.0.deb && \
-  apt-get install ./sbt-1.8.0.deb
+    apt-get install ./sbt-1.8.0.deb -y
 
-RUN apt-get update && apt-get install -y \
+RUN rm sbt-1.8.0.deb
+
+RUN apt-get update && apt-get upgrade -y
+
+RUN apt-get install -y \
     unzip \
-    python-dev \
-    python-pip \
-    libblas-dev \
-    liblapack-dev \
-    gfortran \
-    python-numpy \
-    python-pandas \
+    python3-dev \
+    python3-pip \
     nodejs && \
   apt-get autoremove && \
   apt-get clean
 
-RUN pip install --upgrade pip
-RUN pip install --upgrade setuptools
-
-WORKDIR /opt
+WORKDIR /home
 
 COPY package.json ./
 COPY requirements.txt ./
 
-RUN pip install -r requirements.txt
+# Python3 dependencies.
+RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install -r requirements.txt
+RUN python3 -m pip install --upgrade setuptools
 
 RUN npm install

README.md

@@ -83,7 +83,7 @@ On Windows, we recommend [Windows Powershell](https://docs.microsoft.com/en-us/p
     Successfully tagged projectsidewalk/web:latest
     WARNING: Image for service web was built because it did not already exist. 
     To rebuild this image you must use `docker-compose build` or `docker-compose up --build`.
-    root@[container-id]:/opt#
+    root@[container-id]:/home#
     ```
 
 1. In a separate terminal, run the command below.
@@ -102,10 +102,10 @@ On Windows, we recommend [Windows Powershell](https://docs.microsoft.com/en-us/p
 
     Running "watch" task
     Waiting...
-    [info] Loading project definition from /opt/project
-    [info] Set current project to sidewalk-webpage (in build file:/opt/)
+    [info] Loading project definition from /home/project
+    [info] Set current project to sidewalk-webpage (in build file:/home/)
     [success] Total time: 78 s, completed Dec 20, 2018 8:06:19 AM
-    [info] Updating {file:/opt/}root...
+    [info] Updating {file:/home/}root...
     [info] Resolving it.geosolutions.jaiext.errordiffusion#jt-errordiffusion;1.0.8 .[info] Resolving org.fusesource.jansi#jansi;1.4 ...
     [info] Done updating.
 
@@ -151,6 +151,7 @@ We recommend the [IntelliJ IDEA](https://www.jetbrains.com/idea/) IDE for develo
         - public/javascripts/Gallery/build
         - public/javascripts/Help/build
         - public/javascripts/Progress/build
+        - public/javascripts/PSMap/build
         - public/javascripts/SVLabel/build
         - public/javascripts/SVValidate/build
 1. We then recommend installing a few plugins. To do so, go to `File -> Settings`. Select the `Plugins` option on the left sidebar and then `Marketplace` (on top menubar). For each of the following plugins, enter their name in the "search area" (textfield next to magnifying glass), find the plugin, and click `Install`: [Play 2 Routes](https://plugins.jetbrains.com/plugin/10053-play-2-routes/), [i18n support](https://plugins.jetbrains.com/plugin/12981-i18n-support/), [HOCON](https://plugins.jetbrains.com/plugin/10481-hocon), and [Scala](https://plugins.jetbrains.com/plugin/1347-scala) (if you haven't already). You will then need to restart IntelliJ to install the plugins.
@@ -173,7 +174,7 @@ Database: sidewalk
 1. If you make any changes to the views or other scala files, these changes will be automatically picked up by `sbt`. You'd need to reload the browser once the compilation finishes. For example, a change to `index.scala.html` file results in:
 
     ```
-    [info] Compiling 1 Scala source to /opt/target/scala-2.10/classes...
+    [info] Compiling 1 Scala source to /home/target/scala-2.10/classes...
     [success] Compiled in 260s
 
     --- (RELOAD) ---
@@ -204,13 +205,3 @@ Database: sidewalk
 
     [success] Compiled in 5s
     ```
-
-## Running the application remotely
-NOTE: This has not been tested in a very long time and may not work.
-
-To run the application remotely,
-
-1. Use Play's dist tool to create jar files of the project (i,e., `activator dist`): https://www.playframework.com/documentation/2.3.x/ProductionDist
-1. Upload the zip file to the web server
-1. SSH into the server and unarchive the zip file (e.g., `unzip filename`).
-1. Run `nohup bin/sidewalk-webpage -Dhttp.port=9000 &` ([reference](http://alvinalexander.com/scala/play-framework-deploying-application-production-server)). Sometimes the application tells you that port 9000 (i.e., default port for a Play app) is taken. To kill an application that is occupying the port, first identify the pid with the netstat command `netstat -tulpn | grep :9000` and then use the `kill` command.

app/controllers/AttributeController.scala

@@ -117,7 +117,7 @@ class AttributeController @Inject() (implicit val env: Environment[User, Session
       val submission = request.body.validate[AttributeFormats.ClusteringSubmission]
       submission.fold(
         errors => {
-          Logger.warn("Failed to parse JSON POST request for multi-user clustering results.")
+          Logger.warn("Failed to parse JSON POST request for single-user clustering results.")
           Logger.info(Json.prettyPrint(request.body))
           Future.successful(BadRequest(Json.obj("status" -> "Error", "message" -> JsError.toFlatJson(errors))))
         },

app/controllers/ForgotPasswordController.scala

@@ -12,7 +12,7 @@ import models.user._
 import play.api.i18n.Messages
 import controllers.headers.ProvidesHeader
 import models.daos.slick.DBTableDefinitions.UserTable
-import play.api.Logger
+import play.api.{Logger, Play}
 import play.api.libs.concurrent.Execution.Implicits._
 import scala.concurrent.Future
 import play.api.libs.mailer._
@@ -57,7 +57,7 @@ class ForgotPasswordController @Inject() (
 
               val resetEmail = Email(
                 Messages("reset.pw.email.reset.title"),
-                "Project Sidewalk <[email protected]>",
+                s"Project Sidewalk <${Play.configuration.getString("noreply-email-address").get}>",
                 Seq(email),
                 bodyHtml = Some(views.html.emails.resetPasswordEmail(user, url).body)
               )

app/controllers/helper/AttributeControllerHelper.scala

@@ -62,7 +62,7 @@ object AttributeControllerHelper {
     for ((userId, i) <- usersToUpdate.view.zipWithIndex) {
       Logger.info(s"Finished ${f"${100.0 * i / nUsers}%1.2f"}% of users, next: $userId.")
       val clusteringOutput =
-        Seq("python", "label_clustering.py", "--key", key, "--user_id", userId).!!
+        Seq("python3", "label_clustering.py", "--key", key, "--user_id", userId).!!
       // Logger.info(clusteringOutput)
     }
     Logger.info("Finshed 100% of users!!\n")
@@ -85,7 +85,7 @@ object AttributeControllerHelper {
     // Runs multi-user clustering within each region.
     for ((regionId, i) <- regionIds.view.zipWithIndex) {
       Logger.info(s"Finished ${f"${100.0 * i / nRegions}%1.2f"}% of regions, next: $regionId.")
-      val clusteringOutput = Seq("python", "label_clustering.py", "--key", key, "--region_id", regionId.toString).!!
+      val clusteringOutput = Seq("python3", "label_clustering.py", "--key", key, "--region_id", regionId.toString).!!
     }
     Logger.info("Finshed 100% of regions!!\n\n")
   }

app/views/clustering.scala.html

@@ -10,8 +10,6 @@ <h1>Clustering labels into attributes</h1>
             <button id="btn-multi-user-clustering">Multi User Clustering</button>
             <button id="btn-both-clustering">Single AND Multi User Clustering</button>
             <br/>
-            <label for="hours-to-go-back">If updating only, (integer) hours to go back:</label>
-            <input id="hours-to-go-back" type="number">
             <p id="clustering-status"></p>
         </div>
     </div>
@@ -22,11 +20,9 @@ <h1>Clustering labels into attributes</h1>
             let singleUserButton = document.getElementById('btn-single-user-clustering');
             singleUserButton.onclick = function() {
                 $( "#clustering-status" ).html( 'Running...' );
-                let hours = document.getElementById('hours-to-go-back').value
-                let hoursParam = hours ? `&hoursCutoff=${hours}` : '';
                 $.ajax({
                     type: "get",
-                    url: "/runClustering?clusteringType=singleUser" + hoursParam,
+                    url: "/runClustering?clusteringType=singleUser",
                     contentType: 'application/json; charset=utf-8',
                     success: function (data) {
                         $( "#clustering-status" ).html(
@@ -66,11 +62,9 @@ <h1>Clustering labels into attributes</h1>
             let bothButton = document.getElementById('btn-both-clustering');
             bothButton.onclick = function() {
                 $( "#clustering-status" ).html( 'Running...' );
-                let hours = document.getElementById('hours-to-go-back').value
-                let hoursParam = hours ? `&hoursCutoff=${hours}` : '';
                 $.ajax({
                     type: "get",
-                    url: "/runClustering?clusteringType=both" + hoursParam,
+                    url: "/runClustering?clusteringType=both",
                     contentType: 'application/json; charset=utf-8',
                     success: function (data) {
                         $( "#clustering-status" ).html(

app/views/explore.scala.html

@@ -10,7 +10,11 @@
 @cityName = @{cityNameShort.getOrElse(Messages(s"city.name.$cityId"))}
 
 @main(title, Some("/explore")) {
-    @navbar(user, Some("/explore"))
+    @if(userRoute.isDefined) {
+        @navbar(user, Some(s"/explore?routeId=${userRoute.get.routeId}&resumeRoute=true"))
+    } else {
+        @navbar(user, Some("/explore"))
+    }
 
     <link rel="stylesheet" href='@routes.Assets.at("javascripts/SVLabel/build/SVLabel.css")'/>
     <link rel="stylesheet" href='@routes.Assets.at("stylesheets/animate.css")'/>

build.sbt

@@ -2,7 +2,7 @@ import play.PlayScala
 
 name := """sidewalk-webpage"""
 
-version := "7.18.1"
+version := "7.18.2"
 
 scalaVersion := "2.10.7"
 

check_streets_for_imagery.py

@@ -1,6 +1,5 @@
 import requests
 import pandas as pd
-from pandas.io.json import json_normalize
 import sys
 import os
 from shapely import wkb
@@ -15,7 +14,7 @@ def write_output(no_imagery_df, curr_street):
 
     # If we aren't done, save the last street we were working on at the end to keep track of our progress.
     if curr_street is not None:
-        no_imagery_df = no_imagery_df.append({'street_edge_id': curr_street.street_edge_id, 'region_id': curr_street.region_id}, ignore_index=True)
+        no_imagery_df = pd.concat([no_imagery_df, pd.DataFrame({'street_edge_id': curr_street.street_edge_id, 'region_id': curr_street.region_id}, index=[0])])
 
     # Convert street_edge_id column from float to int.
     no_imagery_df.street_edge_id = no_imagery_df.street_edge_id.astype('int32')
@@ -33,11 +32,12 @@ def redistribute_vertices(geom):
         num_vert = 1
     return LineString([geom.interpolate(float(n) / num_vert, normalized=True) for n in range(num_vert + 1)])
 
-if __name__ == '__main__':
+
+def main():
     # Read google maps API key from env variable.
     api_key = os.getenv('GOOGLE_MAPS_API_KEY')
     if api_key is None:
-        print "Couldn't read GOOGLE_MAPS_API_KEY environment variable."
+        print("Couldn't read GOOGLE_MAPS_API_KEY environment variable.")
         exit(1)
 
     # Read street edge data from CSV.
@@ -47,7 +47,7 @@ def redistribute_vertices(geom):
     street_data['id'] = range(1, n_streets + 1)
 
     # Convert geom to Shapely format and add vertices approximately every 15 meters.
-    street_data['geom'] = map(lambda g: redistribute_vertices(wkb.loads(g, hex=True)), street_data['geom'].values)
+    street_data['geom'] = list(map(lambda g: redistribute_vertices(wkb.loads(g, hex=True)), list(street_data['geom'])))
 
     # Create dataframe that will hold output data.
     streets_with_no_imagery = pd.DataFrame(columns=['street_edge_id', 'region_id'])
@@ -80,13 +80,13 @@ def redistribute_vertices(geom):
         except (requests.exceptions.RequestException, KeyboardInterrupt) as e:
             write_output(streets_with_no_imagery, street)
             exit(1)
-        first_endpoint_fail = json_normalize(first_endpoint.json()).status[0] == 'ZERO_RESULTS'
-        second_endpoint_fail = json_normalize(second_endpoint.json()).status[0] == 'ZERO_RESULTS'
+        first_endpoint_fail = pd.json_normalize(first_endpoint.json()).status[0] == 'ZERO_RESULTS'
+        second_endpoint_fail = pd.json_normalize(second_endpoint.json()).status[0] == 'ZERO_RESULTS'
 
         # If no imagery at either endpoint, add to no imagery list and move on. If at least one has imagery, check many
         # points along the street for imagery to figure out whether or not most of the street is missing imagery.
         if first_endpoint_fail and second_endpoint_fail:
-            streets_with_no_imagery = streets_with_no_imagery.append({'street_edge_id': street.street_edge_id, 'region_id': street.region_id}, ignore_index=True)
+            streets_with_no_imagery = pd.concat([streets_with_no_imagery, pd.DataFrame({'street_edge_id': street.street_edge_id, 'region_id': street.region_id}, index=[0])])
         else:
             n_success = 0
             n_fail = 0
@@ -101,7 +101,7 @@ def redistribute_vertices(geom):
                 except (requests.exceptions.RequestException, KeyboardInterrupt) as e:
                     write_output(streets_with_no_imagery, street)
                     exit(1)
-                response_status = json_normalize(response.json()).status[0]
+                response_status = pd.json_normalize(response.json()).status[0]
                 if response_status == 'ZERO_RESULTS':
                     n_fail += 1
                 else:
@@ -112,9 +112,14 @@ def redistribute_vertices(geom):
                 # looping through the points we meet one of those criteria (or we find that we will not be able to meet
                 # either criteria because there have already been enough points with imagery), we break out of the loop.
                 if n_fail >= 0.5 * n_coord or (n_fail >= 0.25 * n_coord and (first_endpoint_fail or second_endpoint_fail)):
-                    streets_with_no_imagery = streets_with_no_imagery.append({'street_edge_id': street.street_edge_id, 'region_id': street.region_id}, ignore_index=True)
+                    streets_with_no_imagery = pd.concat([streets_with_no_imagery, pd.DataFrame({'street_edge_id': street.street_edge_id, 'region_id': street.region_id}, index=[0])])
                     break
                 elif n_success > 0.75 * n_coord or (n_success > 0.5 * n_coord and not first_endpoint_fail and not second_endpoint_fail):
                     break
+    print() # Stops the overflow on new line
 
     write_output(streets_with_no_imagery, None)
+
+
+if __name__ == '__main__':
+    main()

conf/application.conf

@@ -127,11 +127,10 @@ scistarter-api-key = ${?SCISTARTER_API_KEY}
 
 internal-api-key = ${?INTERNAL_API_KEY}
 
-smtp.host="smtp-relay.brevo.com"
-smtp.port=587
-smtp.tls=true
-smtp.user=${?SIDEWALK_EMAIL_ADDRESS}
-smtp.password=${?SIDEWALK_EMAIL_PASSWORD}
+smtp.host="localhost"
+smtp.port=25
+smtp.tls=false
+noreply-email-address="[email protected]"
 
 # For the ability to save crops on the server
 play.http.parser.maxDiskBuffer = 100MB

conf/evolutions/default/215.sql

@@ -0,0 +1,5 @@
+# --- !Ups
+INSERT INTO version VALUES ('7.18.2', now(), 'Fixes password reset emails not sending and API data not updating.');
+
+# --- !Downs
+DELETE FROM version WHERE version_id = '7.18.2';

docker-compose.yml

@@ -11,17 +11,15 @@ services:
     depends_on:
       - db
     volumes:
-      - "./:/opt/:delegated"
-      - "/opt/node_modules"
+      - "./:/home/:delegated"
+      - "/home/node_modules"
     ports:
       - "9000:9000"
     platform: linux/x86_64
     # platform: linux/arm64
     environment:
       - DATABASE_USER=sidewalk
       - SIDEWALK_CITY_ID=DUMMY_CITY_ID
-      - SIDEWALK_EMAIL_ADDRESS=DUMMY_EMAIL_ADDRESS
-      - SIDEWALK_EMAIL_PASSWORD=DUMMY_EMAIL_PASSWORD
       - MAPBOX_API_KEY=DUMMY_MAPBOX_API_KEY
       - GOOGLE_MAPS_API_KEY=DUMMY_GOOGLE_API_KEY
       - GOOGLE_MAPS_SECRET=DUMMY_GOOGLE_SECRET