Mirrors/lodash
1
0
Fork 0
mirror of https://github.com/whoisclebs/lodash.git synced 2026-01-29 06:27:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement asciiWords and unicodeWords using String.match instead of binding RegExp.exec (#4417)

Browse Source 
* Enable words tests
* Implement asciiWords and unicodeWords using String.match instead of binding RegExp.exec
This commit is contained in:
Luiz Américo
2019-08-18 15:24:56 -03:00
committed by John-David Dalton
parent 15e1557b2a
commit e2555a43ad
3 changed files with 45 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
.internal/unicodeWords.js
View File

@@ -45,14 +45,7 @@ const rsOrdUpper = '\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])'
const rsSeq = rsOptVar + reOptMod + rsOptJoin
const rsEmoji = `(?:${[rsDingbat, rsRegional, rsSurrPair].join('|')})${rsSeq}`
/**
* Splits a Unicode `string` into an array of its words.
*
* @private
* @param {string} The string to inspect.
* @returns {Array} Returns the words of `string`.
*/
const unicodeWords = RegExp.prototype.exec.bind(RegExp([
const reUnicodeWords = RegExp([
`${rsUpper}?${rsLower}+${rsOptContrLower}(?=${[rsBreak, rsUpper, '$'].join('|')})`,
`${rsMiscUpper}+${rsOptContrUpper}(?=${[rsBreak, rsUpper + rsMiscLower, '$'].join('|')})`,
`${rsUpper}?${rsMiscLower}+${rsOptContrLower}`,
@@ -61,6 +54,17 @@ const unicodeWords = RegExp.prototype.exec.bind(RegExp([
rsOrdLower,
`${rsDigit}+`,
rsEmoji
].join('|'), 'g'))
].join('|'), 'g')
/**
* Splits a Unicode `string` into an array of its words.
*
* @private
* @param {string} The string to inspect.
* @returns {Array} Returns the words of `string`.
*/
function unicodeWords(string) {
return string.match(reUnicodeWords)
}
export default unicodeWords

56
test/words.js → test/words.test.js
View File

@@ -1,6 +1,7 @@
import assert from 'assert';
import lodashStable from 'lodash';
import { burredLetters, _, stubArray } from './utils.js';
import words from '../words.js'
describe('words', function() {
it('should match words containing Latin Unicode letters', function() {
@@ -9,36 +10,36 @@ describe('words', function() {
});
var actual = lodashStable.map(burredLetters, function(letter) {
return _.words(letter);
return words(letter);
});
assert.deepStrictEqual(actual, expected);
});
it('should support a `pattern`', function() {
assert.deepStrictEqual(_.words('abcd', /ab|cd/g), ['ab', 'cd']);
assert.deepStrictEqual(_.words('abcd', 'ab|cd'), ['ab']);
assert.deepStrictEqual(words('abcd', /ab|cd/g), ['ab', 'cd']);
assert.deepStrictEqual(Array.from(words('abcd', 'ab|cd')), ['ab']);
});
it('should work with compound words', function() {
assert.deepStrictEqual(_.words('12ft'), ['12', 'ft']);
assert.deepStrictEqual(_.words('aeiouAreVowels'), ['aeiou', 'Are', 'Vowels']);
assert.deepStrictEqual(_.words('enable 6h format'), ['enable', '6', 'h', 'format']);
assert.deepStrictEqual(_.words('enable 24H format'), ['enable', '24', 'H', 'format']);
assert.deepStrictEqual(_.words('isISO8601'), ['is', 'ISO', '8601']);
assert.deepStrictEqual(_.words('LETTERSAeiouAreVowels'), ['LETTERS', 'Aeiou', 'Are', 'Vowels']);
assert.deepStrictEqual(_.words('tooLegit2Quit'), ['too', 'Legit', '2', 'Quit']);
assert.deepStrictEqual(_.words('walk500Miles'), ['walk', '500', 'Miles']);
assert.deepStrictEqual(_.words('xhr2Request'), ['xhr', '2', 'Request']);
assert.deepStrictEqual(_.words('XMLHttp'), ['XML', 'Http']);
assert.deepStrictEqual(_.words('XmlHTTP'), ['Xml', 'HTTP']);
assert.deepStrictEqual(_.words('XmlHttp'), ['Xml', 'Http']);
assert.deepStrictEqual(words('12ft'), ['12', 'ft']);
assert.deepStrictEqual(words('aeiouAreVowels'), ['aeiou', 'Are', 'Vowels']);
assert.deepStrictEqual(words('enable 6h format'), ['enable', '6', 'h', 'format']);
assert.deepStrictEqual(words('enable 24H format'), ['enable', '24', 'H', 'format']);
assert.deepStrictEqual(words('isISO8601'), ['is', 'ISO', '8601']);
assert.deepStrictEqual(words('LETTERSAeiouAreVowels'), ['LETTERS', 'Aeiou', 'Are', 'Vowels']);
assert.deepStrictEqual(words('tooLegit2Quit'), ['too', 'Legit', '2', 'Quit']);
assert.deepStrictEqual(words('walk500Miles'), ['walk', '500', 'Miles']);
assert.deepStrictEqual(words('xhr2Request'), ['xhr', '2', 'Request']);
assert.deepStrictEqual(words('XMLHttp'), ['XML', 'Http']);
assert.deepStrictEqual(words('XmlHTTP'), ['Xml', 'HTTP']);
assert.deepStrictEqual(words('XmlHttp'), ['Xml', 'Http']);
});
it('should work with compound words containing diacritical marks', function() {
assert.deepStrictEqual(_.words('LETTERSÆiouAreVowels'), ['LETTERS', 'Æiou', 'Are', 'Vowels']);
assert.deepStrictEqual(_.words('æiouAreVowels'), ['æiou', 'Are', 'Vowels']);
assert.deepStrictEqual(_.words('æiou2Consonants'), ['æiou', '2', 'Consonants']);
assert.deepStrictEqual(words('LETTERSÆiouAreVowels'), ['LETTERS', 'Æiou', 'Are', 'Vowels']);
assert.deepStrictEqual(words('æiouAreVowels'), ['æiou', 'Are', 'Vowels']);
assert.deepStrictEqual(words('æiou2Consonants'), ['æiou', '2', 'Consonants']);
});
it('should not treat contractions as separate words', function() {
@@ -48,7 +49,7 @@ describe('words', function() {
lodashStable.times(2, function(index) {
var actual = lodashStable.map(postfixes, function(postfix) {
var string = 'a b' + apos + postfix + ' c';
return _.words(string[index ? 'toUpperCase' : 'toLowerCase']());
return words(string[index ? 'toUpperCase' : 'toLowerCase']());
});
var expected = lodashStable.map(postfixes, function(postfix) {
@@ -71,8 +72,8 @@ describe('words', function() {
return [ordinal[index ? 'toUpperCase' : 'toLowerCase']()];
});
var actual = lodashStable.map(expected, function(words) {
return _.words(words[0]);
var actual = lodashStable.map(expected, function(expectedWords) {
return words(expectedWords[0]);
});
assert.deepStrictEqual(actual, expected);
@@ -82,7 +83,7 @@ describe('words', function() {
it('should not treat mathematical operators as words', function() {
var operators = ['\xac', '\xb1', '\xd7', '\xf7'],
expected = lodashStable.map(operators, stubArray),
actual = lodashStable.map(operators, _.words);
actual = lodashStable.map(operators, words);
assert.deepStrictEqual(actual, expected);
});
@@ -95,25 +96,18 @@ describe('words', function() {
];
var expected = lodashStable.map(marks, stubArray),
actual = lodashStable.map(marks, _.words);
actual = lodashStable.map(marks, words);
assert.deepStrictEqual(actual, expected);
});
it('should work as an iteratee for methods like `_.map`', function() {
var strings = lodashStable.map(['a', 'b', 'c'], Object),
actual = lodashStable.map(strings, _.words);
assert.deepStrictEqual(actual, [['a'], ['b'], ['c']]);
});
it('should prevent ReDoS', function() {
var largeWordLen = 50000,
largeWord = 'A'.repeat(largeWordLen),
maxMs = 1000,
startTime = lodashStable.now();
assert.deepStrictEqual(_.words(largeWord + 'ÆiouAreVowels'), [largeWord, 'Æiou', 'Are', 'Vowels']);
assert.deepStrictEqual(words(largeWord + 'ÆiouAreVowels'), [largeWord, 'Æiou', 'Are', 'Vowels']);
var endTime = lodashStable.now(),
timeSpent = endTime - startTime;

11
words.js
View File

@@ -1,13 +1,16 @@
import unicodeWords from './.internal/unicodeWords.js'
const asciiWords = RegExp.prototype.exec.bind(
/[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g
)
const hasUnicodeWord = RegExp.prototype.test.bind(
/[a-z][A-Z]|[A-Z]{2,}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/
)
/** Used to match words composed of alphanumeric characters. */
const reAsciiWord = /[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g
function asciiWords(string) {
return string.match(reAsciiWord)
}
/**
* Splits `string` into an array of its words.
*