Initial commit for twitter-text 3.0.0 (twitter#265)

* Initial commit for twitter-text 3.0.0

* remove js from travis test

* package version change

.travis.yml

@@ -7,6 +7,5 @@ matrix:
     - env: TWITTER_TEXT_DIR=rb
       rvm: 2.4.2
     - env: TWITTER_TEXT_DIR=java
-    - env: TWITTER_TEXT_DIR=js
     - env: TWITTER_TEXT_DIR=objc
 script: "./.travis.sh"

README.md

@@ -1,5 +1,3 @@
-[![Build Status](https://img.shields.io/travis/twitter/twitter-text/master.svg)](https://travis-ci.org/twitter/twitter-text) [![Maven Central](https://img.shields.io/maven-central/v/com.twitter.twittertext/twitter-text.svg)](http://search.maven.org/#search%7Cgav%7C1%7Cg%3A%22com.twitter.twittertext%22%20AND%20a%3A%22twitter-text%22) [![Gem](https://img.shields.io/gem/v/twitter-text.svg)](https://rubygems.org/gems/twitter-text) [![npm](https://img.shields.io/npm/v/twitter-text.svg)](https://www.npmjs.com/package/twitter-text) [![CocoaPods](https://img.shields.io/cocoapods/v/twitter-text.svg)](http://cocoapods.org/?q=twitter-text) [![Bower](https://img.shields.io/bower/v/twitter-text.svg)](http://bower.io/search/?q=twitter-text)
-
 twitter-text
 ============
 

config/README.md

@@ -13,6 +13,7 @@ The configuration format is a JSON string. The JSON can have the following prope
 * `maxWeightedTweetLength` (required, integer, min value 0)
 * `scale` (required, integer, min value 1)
 * `defaultWeight` (required, integer, min value 0)
+* `emojiParsingEnabled` (optional, boolean)
 * `transformedURLLength` (integer, min value 0)
 * `ranges` (array of range items)
 
@@ -48,6 +49,13 @@ The Tweet length is the (`weighted length` / `scale`).
 The default weight applied to all code points. This is overridden in
 one or more range items.
 
+### emojiParsingEnabled
+
+When set to true, the weighted Tweet length considers all emoji as a
+single code point (with a default weight of 200), including longer
+grapheme clusters combined by zero-width joiners. When set to false,
+Tweet length is calculated by weighing individual Unicode code points.
+
 ### transformedURLLength
 
 The length counted for URLs against the total weight of the Tweet. In

config/v3.json

@@ -0,0 +1,30 @@
+{
+  "version": 3,
+  "maxWeightedTweetLength": 280,
+  "scale": 100,
+  "defaultWeight": 200,
+  "emojiParsingEnabled": true,
+  "transformedURLLength": 23,
+  "ranges": [
+    {
+      "start": 0,
+      "end": 4351,
+      "weight": 100
+    },
+    {
+      "start": 8192,
+      "end": 8205,
+      "weight": 100
+    },
+    {
+      "start": 8208,
+      "end": 8223,
+      "weight": 100
+    },
+    {
+      "start": 8242,
+      "end": 8247,
+      "weight": 100
+    }
+  ]
+}

conformance/Rakefile

@@ -39,7 +39,10 @@ package com.twitter.twittertext;
 import java.util.Arrays;
 import java.util.List;
 
-public class TldLists {
+public final class TldLists {
+  private TldLists() {
+  }
+
   public static final List<String> GTLDS = Arrays.asList(
 #{yml["generic"].map {|el| "      \"#{el}\""}.join(",\n")}
   );

conformance/extract.yml

@@ -664,6 +664,44 @@ tests:
         - url: "http://foobar.پاکستان/"
           indices: [42, 64]
 
+  urls_with_directional_markers:
+    - description: "Extract URLs from RTL text"
+      text: "\U00002066\U0000202Atest abcdef.com پاکستان http://twitter.com/\U0000202C\U00002069"
+      expected:
+        - url: "abcdef.com"
+          indices: [7, 17]
+        - url: "http://twitter.com/"
+          indices: [26, 45]
+
+    - description: "Extract URLs from RTL text with embedded directional marks"
+      text: "This is a test \U00002066\U0000202Atwitter.com\U0000202C\U00002069 \U00002066\U0000202Ahttp://foobar.پاکستان/\U0000202C\U00002069⁩ قطر فلسطين عمان"
+      expected:
+        - url: "twitter.com"
+          indices: [17, 28]
+        - url: "http://foobar.پاکستان/"
+          indices: [33, 55]
+
+  tco_urls_with_params:
+    - description: "Extract valid URL with params: https://t.co/UqIyJAJTfo?amp=1"
+      text: "text https://t.co/UqIyJAJTfo?amp=1"
+      expected: ["https://t.co/UqIyJAJTfo?amp=1"]
+
+    - description: "Extract valid URL with params: https://t.co/UqIyJAJTfo?type=js"
+      text: "text https://t.co/UqIyJAJTfo?type=js"
+      expected: ["https://t.co/UqIyJAJTfo?type=js"]
+
+    - description: "Extract valid URL with params: https://t.co/UqIyJAJTfo?ssr=true"
+      text: "text https://t.co/UqIyJAJTfo?ssr=true"
+      expected: ["https://t.co/UqIyJAJTfo?ssr=true"]
+
+    - description: "Extract a valid URL with params: https://t.co/asdfdf?a=b#123"
+      text: "text https://t.co/asdfdf?a=b#123"
+      expected: ["https://t.co/asdfdf?a=b#123"]
+
+    - description: "Extract a valid URL with params: https://t.co/sadfasdf?a=b&c=d"
+      text: "text https://t.co/sadfasdf?a=b&c=d"
+      expected: ["https://t.co/sadfasdf?a=b&c=d"]
+
   hashtags:
     - description: "Extract hashtag after emoji without variant selector (uFE0E or uFE0F)"
       text: "a ✌#hashtag here"

conformance/tld_lib.yml

@@ -343,6 +343,7 @@ generic:
 - 新闻
 - 政府
 - 政务
+- 招聘
 - 手表
 - 手机
 - 我爱你
@@ -598,6 +599,7 @@ generic:
 - srl
 - spreadbetting
 - spot
+- sport
 - spiegel
 - space
 - soy
@@ -942,6 +944,7 @@ generic:
 - locker
 - loans
 - loan
+- llc
 - lixil
 - living
 - live
@@ -1047,6 +1050,7 @@ generic:
 - info
 - infiniti
 - industries
+- inc
 - immobilien
 - immo
 - imdb
@@ -1364,6 +1368,7 @@ generic:
 - cheap
 - chat
 - chase
+- charity
 - channel
 - chanel
 - cfd