Merge pull request #156 from khoa-nd/master

Merged
chrisshayan · Jan 30, 2015 · 3be847a · 3be847a
2 parents 9640490 + 384e1b2
commit 3be847a
Show file tree

Hide file tree

Showing 17 changed files with 1,109 additions and 136 deletions.
diff --git a/src/main/java/com/techlooper/bootcamp/BootcampController.java b/src/main/java/com/techlooper/bootcamp/BootcampController.java
@@ -24,7 +24,7 @@ public class BootcampController {
   @ResponseBody
   @RequestMapping(value = "/bootcamp/users", method = RequestMethod.POST)
   public void save(@RequestBody BootcampUserInfo bootcampUserInfo, HttpServletResponse httpServletResponse) throws IOException {
-    File jsonFile = new File("bootcamp-users.json");
+    File jsonFile = new File("c:/bootcamp-users.json");
     StringBuilder jsonBuilder = new StringBuilder();
     if (jsonFile.exists()) {
       Files.readLines(jsonFile, StandardCharsets.UTF_8).stream().forEach(jsonBuilder::append);

diff --git a/src/main/java/com/techlooper/controller/UserController.java b/src/main/java/com/techlooper/controller/UserController.java
@@ -1,12 +1,16 @@
 package com.techlooper.controller;
 
+import com.techlooper.model.SocialProvider;
 import com.techlooper.model.SocialRequest;
 import com.techlooper.model.UserImportData;
 import com.techlooper.model.UserInfo;
+import com.techlooper.service.SocialService;
+import com.techlooper.service.UserImportDataProcessor;
 import com.techlooper.service.UserService;
 import com.techlooper.util.EmailValidator;
 import org.apache.commons.lang3.StringUtils;
 import org.jasypt.util.text.TextEncryptor;
+import org.springframework.context.ApplicationContext;
 import org.springframework.messaging.handler.annotation.MessageMapping;
 import org.springframework.messaging.simp.annotation.SendToUser;
 import org.springframework.stereotype.Controller;
@@ -26,6 +30,9 @@
 @Controller
 public class UserController {
 
+  @Resource
+  private ApplicationContext applicationContext;
+
   @Resource
   private UserService userService;
 
@@ -42,10 +49,16 @@ public void save(@RequestBody UserImportData userImportData, HttpServletResponse
   @ResponseBody
   @RequestMapping(value = "/api/users/addAll", method = RequestMethod.POST)
   public void saveAll(@RequestBody List<UserImportData> users, HttpServletResponse httpServletResponse) {
-    //TODO : Temporarily adding fake email for user who is missing email address to pass validation
-    processEmailUser(users);
-    httpServletResponse.setStatus(userService.addCrawledUserAll(users) > 0 ?
-            HttpServletResponse.SC_NO_CONTENT : HttpServletResponse.SC_NOT_ACCEPTABLE);
+    if (!users.isEmpty()) {
+      SocialProvider provider = users.get(0).getCrawlerSource();
+      UserImportDataProcessor dataProcessor = applicationContext.getBean(provider + "UserImportDataProcessor", UserImportDataProcessor.class);
+      // process raw user data before import into ElasticSearch
+      dataProcessor.process(users);
+      httpServletResponse.setStatus(userService.addCrawledUserAll(users) > 0 ?
+              HttpServletResponse.SC_NO_CONTENT : HttpServletResponse.SC_NOT_ACCEPTABLE);
+    } else {
+      httpServletResponse.setStatus(HttpServletResponse.SC_NOT_ACCEPTABLE);
+    }
   }
 
   @ResponseBody
@@ -79,14 +92,4 @@ public void verifyUserLogin(@RequestBody SocialRequest searchRequest, @CookieVal
     }
   }
 
-  private void processEmailUser(List<UserImportData> users) {
-    Iterator<UserImportData> iterator = users.iterator();
-    while (iterator.hasNext()) {
-      UserImportData user = iterator.next();
-      if (StringUtils.isEmpty(user.getEmail()) || !EmailValidator.validate(user.getEmail())) {
-        user.setOriginalEmail(user.getEmail());
-        user.setEmail(user.getUsername() + "@missing.com");
-      }
-    }
-  }
 }
diff --git a/src/main/java/com/techlooper/model/UserImportData.java b/src/main/java/com/techlooper/model/UserImportData.java
@@ -1,31 +1,83 @@
 package com.techlooper.model;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.ArrayList;
 import java.util.List;
 
 /**
  * Created by khoa-nd on 27/01/15.
  */
 public class UserImportData {
 
+  @JsonProperty("email")
   private String email;
 
   private String originalEmail;
 
+  @JsonProperty("fullname")
   private String fullName;
 
+  @JsonProperty("crawlersource")
   private SocialProvider crawlerSource;
 
+  @JsonProperty("username")
   private String username;
 
+  @JsonProperty("imageurl")
   private String imageUrl;
 
+  @JsonProperty("location")
   private String location;
 
+  @JsonProperty("datejoin")
   private String dateJoin;
 
+  @JsonProperty("company")
   private String company;
 
-  private List<String> skills;
+  @JsonProperty("website")
+  private String website;
+
+  @JsonProperty("description")
+  private String description;
+
+  @JsonProperty("followers")
+  private String followers;
+
+  @JsonProperty("following")
+  private String following;
+
+  @JsonProperty("organizations")
+  private List<String> organizations;
+
+  @JsonProperty("popular_repos")
+  private List<String> popularRepositories;
+
+  @JsonProperty("contributed_repos")
+  private List<String> contributedRepositories;
+
+  @JsonProperty("contribute_longest_streak_total")
+  private String contributedLongestStreakTotal;
+
+  @JsonProperty("last_contributed_datetime")
+  private String lastContributedDateTime;
+
+  @JsonProperty("contribute_number_last_year")
+  private String contributeNumberLastYear;
+
+  @JsonProperty("contribute_number_last_year_period")
+  private String contributeNumberLastYearPeriod;
+
+  @JsonProperty("contribute_longest_streak_period")
+  private String contributeLongestStreakPeriod;
+
+  @JsonProperty("contribute_current_streak_total")
+  private String contributeCurrentStreakTotal;
+
+  private List<String> skills = new ArrayList<>();
+
+  private int numberOfRepositories;
 
   public String getEmail() {
     return email;
@@ -99,11 +151,123 @@ public void setCompany(String company) {
     this.company = company;
   }
 
+  public String getWebsite() {
+    return website;
+  }
+
+  public void setWebsite(String website) {
+    this.website = website;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public void setDescription(String description) {
+    this.description = description;
+  }
+
+  public String getFollowers() {
+    return followers;
+  }
+
+  public void setFollowers(String followers) {
+    this.followers = followers;
+  }
+
+  public String getFollowing() {
+    return following;
+  }
+
+  public void setFollowing(String following) {
+    this.following = following;
+  }
+
+  public List<String> getOrganizations() {
+    return organizations;
+  }
+
+  public void setOrganizations(List<String> organizations) {
+    this.organizations = organizations;
+  }
+
+  public List<String> getPopularRepositories() {
+    return popularRepositories;
+  }
+
+  public void setPopularRepositories(List<String> popularRepositories) {
+    this.popularRepositories = popularRepositories;
+  }
+
+  public List<String> getContributedRepositories() {
+    return contributedRepositories;
+  }
+
+  public void setContributedRepositories(List<String> contributedRepositories) {
+    this.contributedRepositories = contributedRepositories;
+  }
+
+  public String getContributedLongestStreakTotal() {
+    return contributedLongestStreakTotal;
+  }
+
+  public void setContributedLongestStreakTotal(String contributedLongestStreakTotal) {
+    this.contributedLongestStreakTotal = contributedLongestStreakTotal;
+  }
+
+  public String getLastContributedDateTime() {
+    return lastContributedDateTime;
+  }
+
+  public void setLastContributedDateTime(String lastContributedDateTime) {
+    this.lastContributedDateTime = lastContributedDateTime;
+  }
+
+  public String getContributeNumberLastYear() {
+    return contributeNumberLastYear;
+  }
+
+  public void setContributeNumberLastYear(String contributeNumberLastYear) {
+    this.contributeNumberLastYear = contributeNumberLastYear;
+  }
+
+  public String getContributeNumberLastYearPeriod() {
+    return contributeNumberLastYearPeriod;
+  }
+
+  public void setContributeNumberLastYearPeriod(String contributeNumberLastYearPeriod) {
+    this.contributeNumberLastYearPeriod = contributeNumberLastYearPeriod;
+  }
+
+  public String getContributeLongestStreakPeriod() {
+    return contributeLongestStreakPeriod;
+  }
+
+  public void setContributeLongestStreakPeriod(String contributeLongestStreakPeriod) {
+    this.contributeLongestStreakPeriod = contributeLongestStreakPeriod;
+  }
+
+  public String getContributeCurrentStreakTotal() {
+    return contributeCurrentStreakTotal;
+  }
+
+  public void setContributeCurrentStreakTotal(String contributeCurrentStreakTotal) {
+    this.contributeCurrentStreakTotal = contributeCurrentStreakTotal;
+  }
+
   public List<String> getSkills() {
     return skills;
   }
 
   public void setSkills(List<String> skills) {
     this.skills = skills;
   }
+
+  public int getNumberOfRepositories() {
+    return numberOfRepositories;
+  }
+
+  public void setNumberOfRepositories(int numberOfRepositories) {
+    this.numberOfRepositories = numberOfRepositories;
+  }
 }
diff --git a/src/main/java/com/techlooper/service/UserImportDataProcessor.java b/src/main/java/com/techlooper/service/UserImportDataProcessor.java
@@ -0,0 +1,12 @@
+package com.techlooper.service;
+
+import com.techlooper.model.UserImportData;
+
+import java.util.List;
+
+/**
+ * Created by NguyenDangKhoa on 1/30/15.
+ */
+public interface UserImportDataProcessor {
+  void process(List<UserImportData> users);
+}
diff --git a/src/main/java/com/techlooper/service/impl/GitHubUserImportDataProcessor.java b/src/main/java/com/techlooper/service/impl/GitHubUserImportDataProcessor.java
@@ -0,0 +1,87 @@
+package com.techlooper.service.impl;
+
+import com.techlooper.model.SocialProvider;
+import com.techlooper.model.UserImportData;
+import com.techlooper.service.UserImportDataProcessor;
+import com.techlooper.util.EmailValidator;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * Created by NguyenDangKhoa on 1/30/15.
+ */
+@Service("GITHUBUserImportDataProcessor")
+public class GitHubUserImportDataProcessor implements UserImportDataProcessor {
+
+  public void process(List<UserImportData> users) {
+    for(UserImportData user : users) {
+      processUserEmail(user);
+      extractUserSkillSetFromDescription(user);
+      extractUserNumberOfRepositories(user);
+      processUserFollowers(user);
+      processUserFollowing(user);
+      processUserContributedLongStreakDays(user);
+      processUserContributedNumberLastYear(user);
+    }
+  }
+
+  private void processUserEmail(UserImportData user) {
+    if (StringUtils.isEmpty(user.getEmail()) || !EmailValidator.validate(user.getEmail())) {
+      user.setOriginalEmail(user.getEmail());
+      user.setEmail(user.getUsername() + "@missing.com");
+    }
+  }
+
+  private void extractUserSkillSetFromDescription(UserImportData user) {
+    if (user.getDescription().startsWith(user.getUsername())) {
+      String[] tokens = StringUtils.split(user.getDescription(), " ");
+      for (String token : tokens) {
+        if (token.contains(EmailValidator.COMMA)) {
+          user.getSkills().add(StringUtils.remove(token, EmailValidator.COMMA));
+        } else if (token.contains(EmailValidator.DOT) && !token.toUpperCase().contains(SocialProvider.GITHUB.toString())) {
+          user.getSkills().add(StringUtils.remove(token, EmailValidator.DOT));
+        }
+      }
+    }
+  }
+
+  private void extractUserNumberOfRepositories(UserImportData user) {
+    int beforeNumberRepoIndex = StringUtils.ordinalIndexOf(user.getDescription(), EmailValidator.WHITE_SPACE, 2);
+    int afterNumberRepoIndex = StringUtils.ordinalIndexOf(user.getDescription(), EmailValidator.WHITE_SPACE, 3);
+    String numRepoStr = StringUtils.substring(user.getDescription(), beforeNumberRepoIndex + 1, afterNumberRepoIndex);
+    if (StringUtils.isNumeric(numRepoStr)) {
+      user.setNumberOfRepositories(Integer.valueOf(numRepoStr));
+    }
+  }
+
+  private void processUserFollowers(UserImportData user) {
+    String followers = user.getFollowers();
+    if (StringUtils.isNotEmpty(followers) && followers.contains("k")) {
+      double numberOfFollowers = Double.valueOf(followers.replace("k", "")) * 1000;
+      user.setFollowers(String.valueOf((int)numberOfFollowers));
+    }
+  }
+
+  private void processUserFollowing(UserImportData user) {
+    String following = user.getFollowing();
+    if (StringUtils.isNotEmpty(following) && following.contains("k")) {
+      double numberOfFollowing = Double.valueOf(following.replace("k", "")) * 1000;
+      user.setFollowers(String.valueOf((int)numberOfFollowing));
+    }
+  }
+
+  private void processUserContributedLongStreakDays(UserImportData user) {
+    user.setContributedLongestStreakTotal(StringUtils.isNotEmpty(user.getContributedLongestStreakTotal()) ?
+            user.getContributedLongestStreakTotal().replace(" days", "") : "");
+    user.setContributeCurrentStreakTotal(StringUtils.isNotEmpty(user.getContributeCurrentStreakTotal()) ?
+            user.getContributeCurrentStreakTotal().replace(" days", "") : "");
+  }
+
+  private void processUserContributedNumberLastYear(UserImportData user) {
+    user.setContributeNumberLastYear(StringUtils.isNotEmpty(user.getContributeNumberLastYear()) ?
+            user.getContributeNumberLastYear().replace(" total", "") : "");
+  }
+
+}