Merge branch 'release/v3.0.0'

Author: nojimage

Changes.md

@@ -1,5 +1,12 @@
 # Changes
 
+## 3.0.0
+
+- Pass twitter-text conformance 3.0.0
+- Change default configuration to v3 (emojiParsingEnabled=true)
+- Add t.co with query string support
+- Add Directional Characters support
+
 ## 2.0.2
 
 - Pass twitter-text conformance 2.0.5
@@ -17,4 +24,4 @@
 - Twtter\Text classes no longer extended Regex class.
 - Deprecated `Validator::isValidTweetText()`, `Validator::getTweetLength()`.
 - `Extractor` constractor no longer accepts `$tweet`
-- `Validator` constractor no longer accepts `$tweet` and `$config`. `Validator` constractor only accepts `Configuration` incetance.
+- `Validator` constractor no longer accepts `$tweet` and `$config`. `Validator` constractor only accepts `Configuration` incetance.

README.md

@@ -69,7 +69,14 @@ composer require nojimage/twitter-text-php
 
 ## Length validation ##
 
-twitter-text 2.0 introduces configuration files that define how Tweets are parsed for length. This allows for backwards compatibility and flexibility going forward. Old-style traditional 140-character parsing is defined by the v1.json configuration file, whereas v2.json is updated for "weighted" Tweets where ranges of Unicode code points can have independent weights aside from the default weight. The sum of all code points, each weighted appropriately, should not exceed the max weighted length.
+twitter-text 3.0 updates the config file with `emojiParsingEnabled` config option.
+When true, twitter-text-php will parse and discount emoji supported by the [Unicode Emoji 11.0](http://www.unicode.org/emoji/charts-11.0) (NOTE: Original [twitter-text](https://github.com/twitter/twitter-text) supported [twemoji library](https://github.com/twitter/twemoji)).
+The length of these emoji will be the default weight (200 or two characters) even if they contain multiple code points combined by zero-width joiners.
+This means that emoji with skin tone and gender modifiers no longer count as more characters than those without such modifiers.
+
+twitter-text 2.0 introduced configuration files that define how Tweets are parsed for length. This allows for backwards compatibility and flexibility going forward.
+Old-style traditional 140-character parsing is defined by the v1.json configuration file, whereas v2.json is updated for "weighted" Tweets where ranges of Unicode code points can have independent weights aside from the default weight.
+The sum of all code points, each weighted appropriately, should not exceed the max weighted length.
 
 Some old methods from twitter-text-php 1.0 have been marked deprecated, such as the `Twitter\Text\Validator::isValidTweetText()`, `Twitter\Text\Validator::getTweetLength()` method. The new API is based on the following method, `Twitter\Text\Parser::parseTweet()`
 

build/build-emoji-regex.php

@@ -0,0 +1,47 @@
+<?php
+
+require dirname(__DIR__) . '/vendor/autoload.php';
+
+$classFile = dirname(__DIR__) . '/lib/Twitter/Text/EmojiRegex.php';
+$emojiDataUrl = 'https://www.unicode.org/Public/emoji/11.0/emoji-test.txt';
+
+// --
+$emojiData = file($emojiDataUrl);
+$emojiCodes = array_map(function ($line) {
+    $value = preg_replace('/^([0-9A-F]{4,}(?: [0-9A-F]{4,})*)\s*;.*$/u', '$1', trim($line));
+
+    return explode(' ', $value);
+}, array_filter($emojiData, function ($line) {
+    return preg_match('/^[0-9A-F]{2}[0-9A-F]{2,}.*; /', $line);
+}));
+
+// sort code length, reverse®
+usort($emojiCodes, function ($a, $b) {
+    $aLength = count($a);
+    $bLength = count($b);
+    if ($aLength === $bLength) {
+        return 0;
+    }
+
+    return ($bLength < $aLength) ? -1 : 1;
+});
+
+$codeRegexList = array_reduce($emojiCodes, function ($carry, $codes) {
+    $carry[] = implode('', array_map(function ($c) {
+        return sprintf('\x{%s}', strtolower($c));
+    }, $codes));
+
+    return $carry;
+}, array());
+
+$regex = implode('|', $codeRegexList);
+
+echo "\n";
+
+// -- modify class file
+$classContent = file_get_contents($classFile);
+$replacedClassContent = preg_replace('/(\s+const VALID_EMOJI_PATTERN = \')(?:.*)(\';)/', '$1/' . $regex . '/u$2', $classContent);
+
+echo $replacedClassContent;
+
+file_put_contents($classFile, $replacedClassContent);

build/build-tld-lists.php

@@ -0,0 +1,96 @@
+<?php
+
+$rootDir = dirname(__DIR__);
+require $rootDir . '/vendor/autoload.php';
+
+use Symfony\Component\Yaml\Yaml;
+
+$classFile = $rootDir . '/lib/Twitter/Text/TldLists.php';
+$tlds = Yaml::parse($rootDir . '/vendor/twitter/twitter-text/conformance/tld_lib.yml');
+
+ob_start();
+echo "<?php\n";
+?>
+
+/**
+ * @author     Takashi Nojima
+ * @copyright  Copyright <?= date('Y') ?>, Takashi Nojima
+ * @license    http://www.apache.org/licenses/LICENSE-2.0  Apache License v2.0
+ * @package    Twitter.Text
+ */
+
+namespace Twitter\Text;
+
+/**
+ * TLD Lists
+ */
+final class TldLists
+{
+    /**
+     * gTLDs
+     *
+     * @var array
+     */
+    private static $gTLDs = array(
+<?php foreach ($tlds['generic'] as $tld) : ?>
+        '<?= $tld ?>',
+<?php endforeach; ?>
+    );
+
+    /**
+     * gTLDs
+     *
+     * @var array
+     */
+    private static $ccTLDs = array(
+<?php foreach ($tlds['country'] as $tld) : ?>
+        '<?= $tld ?>',
+<?php endforeach; ?>
+    );
+
+    /**
+     * get valid gTLD regexp
+     *
+     * @staticvar string $regex
+     * @return string
+     */
+    final public static function getValidGTLD()
+    {
+        static $regex;
+
+        if (!empty($regex)) {
+            return $regex;
+        }
+
+        $gTLD = implode('|', static::$gTLDs);
+        $regex = '(?:(?:' . $gTLD . ')(?=[^0-9a-z@]|$))';
+
+        return $regex;
+    }
+
+    /**
+     * get valid ccTLD regexp
+     *
+     * @staticvar string $regex
+     * @return string
+     */
+    final public static function getValidCcTLD()
+    {
+        static $regex;
+
+        if (!empty($regex)) {
+            return $regex;
+        }
+
+        $ccTLD = implode('|', static::$ccTLDs);
+        $regex = '(?:(?:' . $ccTLD . ')(?=[^0-9a-z@]|$))';
+
+        return $regex;
+    }
+}
+<?php
+$content = ob_get_clean();
+
+echo $content;
+
+file_put_contents($classFile, $content);

composer.json

@@ -31,11 +31,11 @@
       "type": "package",
       "package": {
         "name": "twitter/twitter-text",
-        "version": "2.0.5",
+        "version": "3.0.0",
         "source": {
           "url": "https://github.com/twitter/twitter-text.git",
           "type": "git",
-          "reference": "v2.0.5"
+          "reference": "v3.0.0"
         }
       }
     }
@@ -46,15 +46,21 @@
     "ext-intl": "*"
   },
   "require-dev": {
+    "ext-json": "*",
     "symfony/yaml": "~2.6.0",
     "phpunit/phpunit": "4.8.*|5.7.*|6.5.*",
-    "twitter/twitter-text": "^2.0.0"
+    "twitter/twitter-text": "^3.0.0"
   },
   "autoload": {
     "psr-0": {
       "Twitter\\Text\\": "lib/"
     }
   },
+  "autoload-dev": {
+    "psr-4": {
+      "Twitter\\Text\\TestCase\\": "tests/TestCase/"
+    }
+  },
   "scripts": {
     "check": [
       "@cs",

lib/Twitter/Text/Autolink.php

@@ -10,10 +10,6 @@
 
 namespace Twitter\Text;
 
-use Twitter\Text\Regex;
-use Twitter\Text\Extractor;
-use Twitter\Text\StringUtils;
-
 /**
  * Twitter Autolink Class
  *
@@ -401,7 +397,7 @@ public function setTarget($v)
      */
     public function autoLinkEntities($tweet = null, $entities = null)
     {
-        if (is_null($tweet)) {
+        if ($tweet === null) {
             $tweet = $this->tweet;
         }
 
@@ -445,9 +441,7 @@ public function autoLinkWithJson($tweet = null, $json = null)
             $json = $this->object2array($json);
         }
         if (is_array($json)) {
-            foreach ($json as $key => $vals) {
-                $entities = array_merge($entities, $json[$key]);
-            }
+            $entities = call_user_func_array('array_merge', $json);
         }
 
         // map JSON entity to twitter-text entity
@@ -487,7 +481,7 @@ protected function object2array($obj)
      */
     public function autoLink($tweet = null)
     {
-        if (is_null($tweet)) {
+        if ($tweet === null) {
             $tweet = $this->tweet;
         }
         $entities = $this->extractor->extractURLWithoutProtocol(false)->extractEntitiesWithIndices($tweet);
@@ -504,7 +498,7 @@ public function autoLink($tweet = null)
      */
     public function autoLinkUsernamesAndLists($tweet = null)
     {
-        if (is_null($tweet)) {
+        if ($tweet === null) {
             $tweet = $this->tweet;
         }
         $entities = $this->extractor->extractMentionsOrListsWithIndices($tweet);
@@ -520,7 +514,7 @@ public function autoLinkUsernamesAndLists($tweet = null)
      */
     public function autoLinkHashtags($tweet = null)
     {
-        if (is_null($tweet)) {
+        if ($tweet === null) {
             $tweet = $this->tweet;
         }
         $entities = $this->extractor->extractHashtagsWithIndices($tweet);
@@ -537,7 +531,7 @@ public function autoLinkHashtags($tweet = null)
      */
     public function autoLinkURLs($tweet = null)
     {
-        if (is_null($tweet)) {
+        if ($tweet === null) {
             $tweet = $this->tweet;
         }
         $entities = $this->extractor->extractURLWithoutProtocol(false)->extractURLsWithIndices($tweet);
@@ -553,7 +547,7 @@ public function autoLinkURLs($tweet = null)
      */
     public function autoLinkCashtags($tweet = null)
     {
-        if (is_null($tweet)) {
+        if ($tweet === null) {
             $tweet = $this->tweet;
         }
         $entities = $this->extractor->extractCashtagsWithIndices($tweet);
@@ -650,7 +644,7 @@ public function linkToUrl($entity)
      */
     public function linkToHashtag($entity, $tweet = null)
     {
-        if (is_null($tweet)) {
+        if ($tweet === null) {
             $tweet = $this->tweet;
         }
 
@@ -668,7 +662,7 @@ public function linkToHashtag($entity, $tweet = null)
             $class[] = 'rtl';
         }
         if (!empty($class)) {
-            $attributes['class'] = join(' ', $class);
+            $attributes['class'] = implode(' ', $class);
         }
 
         return $this->linkToText($entity, $linkText, $attributes);
@@ -712,7 +706,7 @@ public function linkToMentionAndList($entity)
      */
     public function linkToCashtag($entity, $tweet = null)
     {
-        if (is_null($tweet)) {
+        if ($tweet === null) {
             $tweet = $this->tweet;
         }
         $attributes = array();
@@ -745,7 +739,7 @@ public function linkToText(array $entity, $text, $attributes = array())
             $rel[] = 'nofollow';
         }
         if (!empty($rel)) {
-            $attributes['rel'] = join(' ', $rel);
+            $attributes['rel'] = implode(' ', $rel);
         }
         if ($this->target) {
             $attributes['target'] = $this->target;