tests:
  emails:
    - description: "Extract emails from beginning of sentence"
      text: "testing+wife@example.org was her address"
      expected: ["testing+wife@example.org"]

    - description: "Extract emails from end of sentence"
      text: "This is a sentence. And this is another testing+wife@example.org"
      expected: ["testing+wife@example.org"]

    - description: "does not extract emails from quote"
      text: "    testing+wife@example.org"
      expected: []

    - description: "Extract emails from mid-sentence"
      text: "Please mr postman pat@royalmail.com look and see"
      expected: ["pat@royalmail.com"]

    - description: "Extract emails separated by newlines"
      text: "\none@example.com\ntwo@example.com\n"
      expected: ["one@example.com", "two@example.com"]

    - description: "email, comma separated"
      text: "one@example.com, two@example.co.uk, three@example.co"
      expected: ["one@example.com", "two@example.co.uk", "three@example.co"]

    - description: "email, comma separated with no trailing space after comma"
      text: "one@example.com,two@example.co.uk,three@example.co"
      expected: ["one@example.com", "two@example.co.uk", "three@example.co"]

    - description: "email, in parentheses"
      text: "(one@example.com)"
      expected: ["one@example.com"]

    - description: "email, followed by a dot"
      text: "one@example.com."
      expected: ["one@example.com"]

    - description: "email with underscore"
      text: "email_lodash@test.com"
      expected: ["email_lodash@test.com"]

    - description: "email with just numbers"
      text: "13@hotmail.com"
      expected: ["13@hotmail.com"]

    - description: "email with a dash"
      text: "email-justdash@test.com"
      expected: ["email-justdash@test.com"]

    - description: "email with a plus"
      text: "email+plus@test.com"
      expected: ["email+plus@test.com"]

  emails_with_indices:
    - description: "Extract lone email with indices"
      text: "hi@hi.com"
      expected:
        - email: "hi@hi.com"
          indices: [0, 9]

    - description: "Extract email with indices inside quotes"
      text: "\"hi@hi.com\""
      expected:
        - email: "hi@hi.com"
          indices: [1, 10]

    - description: "Extract email with indices inside parentheses"
      text: "Hello (hi@hi.com)"
      expected:
        - email: "hi@hi.com"
          indices: [7, 16]

    - description: "Extract email with indices inside single quotes"
      text: "'hi@hi.com'"
      expected:
        - email: "hi@hi.com"
          indices: [1, 10]

    - description: "Extract lone email followed by a comma"
      text: "hi@hi.com,"
      expected:
        - email: "hi@hi.com"
          indices: [0, 9]

    - description: "Extract email with indices from beginning of sentence"
      text: "hi@hi.com hi2u!"
      expected:
        - email: "hi@hi.com"
          indices: [0, 9]

    - description: "Extract email with indices from end of sentence"
      text: "hi2u hi@hi.com"
      expected:
        - email: "hi@hi.com"
          indices: [5, 14]

    - description: "Extract emails with indices from middle of sentence"
      text: "hi2u hi@hi.com dude"
      expected:
        - email: "hi@hi.com"
          indices: [5, 14]

    - description: "Extract multiple emails with indices"
      text: "hi@hi.com ho@hi.com"
      expected:
        - email: "hi@hi.com"
          indices: [0, 9]
        - email: "ho@hi.com"
          indices: [10, 19]

    - description: "Extract multiple same emails with indices"
      text: "hi@hi.com hi@hi.com"
      expected:
        - email: "hi@hi.com"
          indices: [0, 9]
        - email: "hi@hi.com"
          indices: [10, 19]

    - description: "Extract multiple emails with indices separated by commas and space"
      text: "hi@hi.com, ho@hi.com"
      expected:
        - email: "hi@hi.com"
          indices: [0, 9]
        - email: "ho@hi.com"
          indices: [11, 20]

    - description: "Extract multiple emails with indices separated by just commas"
      text: "hi@hi.com,ho@hi.com"
      expected:
        - email: "hi@hi.com"
          indices: [0, 9]
        - email: "ho@hi.com"
          indices: [10, 19]

  highlight:
    - description: "match nickname without @ in the middle of string"
      text: "aa nick foo"
      nick: "nick"
      expected: true

    - description: "match nickname in beginning of string"
      text: "nick foo"
      nick: "nick"
      expected: true

    - description: "match nickname in end of string"
      text: "foo nick"
      nick: "nick"
      expected: true

    - description: "match nickname after non word character"
      text: "!nick"
      nick: "nick"
      expected: true

    - description: "match nickname before non word"
      text: "nick!"
      nick: "nick"
      expected: true

    - description: "no nick is no nick"
      text: "no no no"
      nick: "nick"
      expected: false

  mentions:
    - description: "mention after invalid mention"
      text: "oo@some @other"
      expected: ["@other"]

    - description: "no mentions from quotes"
      text: "@some\n    but not this @other"
      expected: ["@some"]

    - description: "Extract mention at the begining of a tweet"
      text: "@username reply"
      expected: ["@username"]

    - description: "Extract mention at the end of a tweet"
      text: "mention @username"
      expected: ["@username"]

    - description: "Extract mention in the middle of a tweet"
      text: "mention @username in the middle"
      expected: ["@username"]

    - description: "Extract mention of username with underscore"
      text: "mention @user_name"
      expected: ["@user_name"]

    - description: "Extract mention of username with dot in middle"
      text: "mention @J.R"
      expected: ["@J.R"]

    - description: "Do not extract a dot at the end of username"
      text: "i'm mentioning @username."
      expected: ["@username"]

    - description: "Extract mention of username with a dash"
      text: "mention @user-name"
      expected: ["@user-name"]

    - description: "Extract mention of all numeric username"
      text: "mention @12345"
      expected: ["@12345"]

    - description: "Extract mention or multiple usernames"
      text: "mention @username1 @username2, @user.name @davy.crokett."
      expected: ["@username1", "@username2", "@user.name", "@davy.crokett"]

    - description: "Extract username followed by accented latin characters"
      text: "@aliceìnheiro something something"
      expected: ["@aliceìnheiro"]

    - description: "DO NOT extract 'http' in '@http://' as username"
      text: "@http://twitter.com"
      expected: []

    - description: "Extract mentions before newline"
      text: "@username\n@mention"
      expected: ["@username", "@mention"]

  mentions_with_predefined_usertags:
    - description: "Include only predefined users"
      text: "@username @foo"
      expected: ["@username"]

  mentions_with_indices:
    - description: "Extract a mention at the start"
      text: "@username yo!"
      expected:
        - tag: "@username"
          indices: [0, 9]

    - description: "Extract a mention that has the same thing mentioned at the start"
      text: "username @username"
      expected:
        - tag: "@username"
          indices: [9, 18]

  mentions_with_indices_with_predefined_usertags:
    - description: "Include only predefined users"
      text: "@username @foobar yo!"
      expected:
        - tag: "@username"
          indices: [0, 9]
  urls:
    - description: "Extracts url in quote block"
      text: "    http://quoted.example.com"
      expected: ["http://quoted.example.com"]

    - description: "Extract a lone URL"
      text: "http://example.com"
      expected: ["http://example.com"]

    - description: "Extract valid URL: http://google.com"
      text: "text http://google.com"
      expected: ["http://google.com"]

    - description: "Extract valid URL: http://foobar.com/#"
      text: "text http://foobar.com/#"
      expected: ["http://foobar.com/#"]

    - description: "Extract valid URL: http://google.com/#foo"
      text: "text http://google.com/#foo"
      expected: ["http://google.com/#foo"]

    - description: "Extract valid URL: http://google.com/#search?q=iphone%20-filter%3Alinks"
      text: "text http://google.com/#search?q=iphone%20-filter%3Alinks"
      expected: ["http://google.com/#search?q=iphone%20-filter%3Alinks"]

    - description: "Extract valid URL: http://twitter.com/#search?q=iphone%20-filter%3Alinks"
      text: "text http://twitter.com/#search?q=iphone%20-filter%3Alinks"
      expected: ["http://twitter.com/#search?q=iphone%20-filter%3Alinks"]

    - description: "Extract valid URL: http://somedomain.com/index.php?path=/abc/def/"
      text: "text http://somedomain.com/index.php?path=/abc/def/"
      expected: ["http://somedomain.com/index.php?path=/abc/def/"]

    - description: "Extract valid URL: http://www.boingboing.net/2007/02/14/katamari_damacy_phon.html"
      text: "text http://www.boingboing.net/2007/02/14/katamari_damacy_phon.html"
      expected: ["http://www.boingboing.net/2007/02/14/katamari_damacy_phon.html"]

    - description: "Extract valid URL: http://somehost.com:3000"
      text: "text http://somehost.com:3000"
      expected: ["http://somehost.com:3000"]

    - description: "Extract ip with protocol"
      text: "http://127.0.0.1"
      expected: ["http://127.0.0.1"]

    - description: "Extract other ip with protocol"
      text: "http://173.194.70.101"
      expected: ["http://173.194.70.101"]

    - description: "Extract ip with port"
      text: "http://127.0.0.1:3000"
      expected: ["http://127.0.0.1:3000"]

    - description: "Extract ip with protocol and path"
      text: "http://127.0.0.1/foobar"
      expected: ["http://127.0.0.1/foobar"]

    - description: "Extract ip with protocol and path ending with a slash"
      text: "http://127.0.0.1/foobar/"
      expected: ["http://127.0.0.1/foobar/"]

    - description: "Extract ip with protocol and query params"
      text: "http://127.0.0.1/?foo=bar"
      expected: ["http://127.0.0.1/?foo=bar"]

    - description: "Extract url with .local domain"
      text: "http://build.local"
      expected: ["http://build.local"]

    - description: "Extract url with .dmz domain"
      text: "http://build.dmz"
      expected: ["http://build.dmz"]

    - description: "Extract url with .lan domain"
      text: "http://build.lan"
      expected: ["http://build.lan"]

    - description: "Extract url with .lan domain and path"
      text: "http://build.lan/yello"
      expected: ["http://build.lan/yello"]

    - description: "Extract url with .technology domain"
      text: "http://build.technology"
      expected: ["http://build.technology"]

    - description: "Extract valid URL: http://xo.com/~matthew+%ff-x"
      text: "text http://xo.com/~matthew+%ff-x"
      expected: ["http://xo.com/~matthew+%ff-x"]

    - description: "Extract valid URL: http://xo.com/~matthew+%ff-,.;x"
      text: "text http://xo.com/~matthew+%ff-,.;x"
      expected: ["http://xo.com/~matthew+%ff-,.;x"]

    - description: "Extract valid URL: http://xo.com/,.;x"
      text: "text http://xo.com/,.;x"
      expected: ["http://xo.com/,.;x"]

    - description: "Extract url with custom TLD"
      text: "http://foo.customtld/path"
      expected: ["http://foo.customtld/path"]

    - description: "Extract valid URL: http://en.wikipedia.org/wiki/Primer_(film)"
      text: "text http://en.wikipedia.org/wiki/Primer_(film)"
      expected: ["http://en.wikipedia.org/wiki/Primer_(film)"]

    - description: "Extract valid URL: http://www.ams.org/bookstore-getitem/item=mbk-59"
      text: "text http://www.ams.org/bookstore-getitem/item=mbk-59"
      expected: ["http://www.ams.org/bookstore-getitem/item=mbk-59"]

    - description: "Extract valid URL: http://✪df.ws/ejp"
      text: "text http://✪df.ws/ejp"
      expected: ["http://✪df.ws/ejp"]

    - description: "Extract valid URL: http://chilp.it/?77e8fd"
      text: "text http://chilp.it/?77e8fd"
      expected: ["http://chilp.it/?77e8fd"]

    - description: "Extract valid URL: http://x.com/oneletterdomain"
      text: "text http://x.com/oneletterdomain"
      expected: ["http://x.com/oneletterdomain"]

    - description: "Extract valid URL: http://msdn.microsoft.com/ja-jp/library/system.net.httpwebrequest(v=VS.100).aspx"
      text: "text http://msdn.microsoft.com/ja-jp/library/system.net.httpwebrequest(v=VS.100).aspx"
      expected: ["http://msdn.microsoft.com/ja-jp/library/system.net.httpwebrequest(v=VS.100).aspx"]

    - description: "DO NOT extract invalid URL: http://domain-begin_dash_2314352345_dfasd.foo-cow_4352.com"
      text: "text http://domain-dash_2314352345_dfasd.foo-cow_4352.com"
      expected: []

    - description: "DO NOT extract invalid URL: http://-begin_dash_2314352345_dfasd.foo-cow_4352.com"
      text: "text http://-dash_2314352345_dfasd.foo-cow_4352.com"
      expected: []

    - description: "Extract a dashed URL without TLD with protocol"
      text: "text http://no-tld"
      expected: ["http://no-tld"]

    - description: "Extract a dashed URL without TLD with protocol and port"
      text: "text http://no-tld:1337"
      expected: ["http://no-tld:1337"]

    - description: "Extract URL without TLD with protocol"
      text: "text http://domain"
      expected: ["http://domain"]

    - description: "Extract URL without TLD with protocol and path"
      text: "text http://domain/foo?bar"
      expected: ["http://domain/foo?bar"]

    - description: "DO NOT extract invalid URL: http://tld-too-short.x"
      text: "text http://tld-too-short.x"
      expected: []

    - description: "Extract a very long hyphenated sub-domain URL (single letter hyphens)"
      text: "text http://word-and-a-number-8-ftw.domain.com/"
      expected: ["http://word-and-a-number-8-ftw.domain.com/"]

    - description: "Extract a hyphenated TLD (usually a typo)"
      text: "text http://domain.com-that-you-should-have-put-a-space-after"
      expected: ["http://domain.com"]

    - description: "Extract URL ending with # value"
      text: "text http://foo.com?#foo text"
      expected: ["http://foo.com?#foo"]

    - description: "Extract URLs without protocol on (com|org|edu|gov|net) domains"
      text: "foo.com foo.net foo.org foo.edu foo.gov"
      expected: ["foo.com", "foo.net", "foo.org", "foo.edu", "foo.gov"]

    - description: "Extract URLs without protocol not on (com|org|edu|gov|net) domains"
      text: "foo.bax foo.co.jp www.foo.bax www.foo.co.uk wwwww.foo foo.comm foo.somecom foo.govedu foo.jp foo.something.fi"
      expected: ["foo.co.jp", "www.foo.co.uk", "foo.jp", "foo.something.fi"]

    - description: "Extract URLs without the protocol with a dash in the subdomain"
      text: "test twitter-dash.com"
      expected: ["twitter-dash.com"]

    - description: "Extract URLs without the protocol and with CCTLD and a dash in the subdomain"
      text: "test twitter-dash.fi"
      expected: ["twitter-dash.fi"]

    - description: "DO NOT Extract URLs without the protocol starting with a dash"
      text: "test -dash.com"
      expected: []

    - description: "DO NOT Extract URLs without the protocol with a subdomain ending to a dash"
      text: "test dash-.domain.com"
      expected: []

    - description: "DO NOT Extract URLs without the protocol ending with a dash"
      text: "test dash-.com"
      expected: []

    - description: "DO NOT Extract URLs without the protocol having only a dash in domain"
      text: "test -.com"
      expected: []

    - description: "Extract URLs without protocol on ccTLD with slash"
      text: "t.co/abcde bit.ly/abcde"
      expected: ["t.co/abcde", "bit.ly/abcde"]

    - description: "Extract URLs with protocol on ccTLD domains"
      text: "http://foo.jp http://fooooo.jp"
      expected: ["http://foo.jp", "http://fooooo.jp"]

    - description: "Extract URLs with a - or + at the end of the path"
      text: "Go to http://example.com/a+ or http://example.com/a-"
      expected: ["http://example.com/a+", "http://example.com/a-"]

    - description: "Extract URLs with longer paths ending in -"
      text: "Go to http://example.com/view/slug-url-?foo=bar"
      expected: ["http://example.com/view/slug-url-?foo=bar"]

    - description: "Extract URLs beginning with a space"
      text: "@user Try http:// example.com/path"
      expected: ["example.com/path"]

    - description: "Extract long URL without protocol surrounded by CJK characters"
      text: "これは日本語です。example.com/path/index.html中国語example.com/path한국"
      expected: ["example.com/path/index.html", "example.com/path"]

    - description: "Extract short URL without protocol surrounded by CJK characters"
      text: "twitter.comこれは日本語です。example.com中国語t.co/abcde한국twitter.com example2.comテストtwitter.com/abcde"
      expected: ["twitter.com", "example.com", "t.co/abcde", "twitter.com", "example2.com", "twitter.com/abcde"]

    - description: "Extract URLs with and without protocol surrounded by CJK characters"
      text: "http://twitter.com/これは日本語です。example.com中国語http://t.co/abcde한국twitter.comテストexample2.comテストhttp://twitter.com/abcde"
      expected: ["http://twitter.com/", "example.com", "http://t.co/abcde", "twitter.com", "example2.com", "http://twitter.com/abcde"]

    - description: "Extract short URLs without protocol on ccTLD domains without path"
      text: "twitter.jp日本語t.co中国語foo.jp t.co foo.jp"
      expected: ["twitter.jp", "t.co", "foo.jp", "t.co", "foo.jp"]

    - description: "Extract URLs beginning with a non-breaking space (U+00A0)"
      text: "@user Try http:// example.com/path"
      expected: ["example.com/path"]

    - description: "Extract URLs with underscores and dashes in the subdomain"
      text: "test http://sub_domain-dash.twitter.com"
      expected: ["http://sub_domain-dash.twitter.com"]

    - description: "Extract URL with minimum number of valid characters"
      text: "test http://a.b.cd"
      expected: ["http://a.b.cd"]

    - description: "Extract URLs containing underscores and dashes"
      text: "test http://a_b.c-d.com"
      expected: ["http://a_b.c-d.com"]

    - description: "Extract URLs containing dashes in the subdomain"
      text: "test http://a-b.c.com"
      expected: ["http://a-b.c.com"]

    - description: "Extract URLs with dashes in the domain name"
      text: "test http://twitter-dash.com"
      expected: ["http://twitter-dash.com"]

    - description: "Extract URLs with lots of symbols then a period"
      text: "http://www.bestbuy.com/site/Currie+Technologies+-+Ezip+400+Scooter/9885188.p?id=1218189013070&skuId=9885188"
      expected: ["http://www.bestbuy.com/site/Currie+Technologies+-+Ezip+400+Scooter/9885188.p?id=1218189013070&skuId=9885188"]

    - description: "DO NOT extract URLs containing leading dashes in the subdomain"
      text: "test http://-leadingdash.twitter.com"
      expected: []

    - description: "DO NOT extract URLs containing trailing dashes in the subdomain"
      text: "test http://trailingdash-.twitter.com"
      expected: []

    - description: "DO NOT extract URLs containing leading underscores in the subdomain"
      text: "test http://_leadingunderscore.twitter.com"
      expected: []

    - description: "DO NOT extract URLs containing trailing underscores in the subdomain"
      text: "test http://trailingunderscore_.twitter.com"
      expected: []

    - description: "DO NOT extract URLs containing leading dashes in the domain name"
      text: "test http://-twitter.com"
      expected: []

    - description: "DO NOT extract URLs containing trailing dashes in the domain name"
      text: "test http://twitter-.com"
      expected: []

    - description: "DO NOT extract URLs containing underscores in the domain name"
      text: "test http://twitter_underscore.com"
      expected: []

    - description: "DO NOT extract URLs containing underscores in the tld"
      text: "test http://twitter.c_o_m"
      expected: []

    - description: "Extract valid URL http://www.foo.com/foo/path-with-period./"
      text: "test http://www.foo.com/foo/path-with-period./"
      expected: ["http://www.foo.com/foo/path-with-period./"]

    - description: "Extract valid URL http://www.foo.org.za/foo/bar/688.1"
      text: "test http://www.foo.org.za/foo/bar/688.1"
      expected: ["http://www.foo.org.za/foo/bar/688.1"]

    - description: "Extract valid URL http://www.foo.com/bar-path/some.stm?param1=foo;param2=P1|0||P2|0"
      text: "test http://www.foo.com/bar-path/some.stm?param1=foo;param2=P1|0||P2|0"
      expected: ["http://www.foo.com/bar-path/some.stm?param1=foo;param2=P1|0||P2|0"]

    - description: "Extract valid URL http://foo.com/bar/123/foo_&_bar/"
      text: "test http://foo.com/bar/123/foo_&_bar/"
      expected: ["http://foo.com/bar/123/foo_&_bar/"]

    - description: "Extract valid URL http://www.cp.sc.edu/events/65"
      text: "test http://www.cp.sc.edu/events/65 test"
      expected: ["http://www.cp.sc.edu/events/65"]

    - description: "Extract valid URL http://www.andersondaradio.no.comunidades.net/"
      text: "http://www.andersondaradio.no.comunidades.net/ test test"
      expected: ["http://www.andersondaradio.no.comunidades.net/"]

    - description: "Extract valid URL ELPAÍS.com"
      text: "test ELPAÍS.com"
      expected: ["ELPAÍS.com"]

    - description: "DO NOT include period at the end of URL"
      text: "test http://twitter.com/."
      expected: ["http://twitter.com/"]

    - description: "Extract a URL with '?' in fragment"
      text: "http://tn.com.ar/show/00056158/la-estrella-del-certamen-el-turno-de-pamela-anderson?fb_xd_fragment#?=&cb=fe17523f223b7&relation=parent.parent&transport=fragment&type=resize&height=20&ackdata"
      expected: ["http://tn.com.ar/show/00056158/la-estrella-del-certamen-el-turno-de-pamela-anderson?fb_xd_fragment#?=&cb=fe17523f223b7&relation=parent.parent&transport=fragment&type=resize&height=20&ackdata"]

    - description: "Extract a URL with '?' in fragment in a text"
      text: "text http://tn.com.ar/show/00056158/la-estrella-del-certamen-el-turno-de-pamela-anderson?fb_xd_fragment#?=&cb=fe17523f223b7&relation=parent.parent&transport=fragment&type=resize&height=20&ackdata text"
      expected: ["http://tn.com.ar/show/00056158/la-estrella-del-certamen-el-turno-de-pamela-anderson?fb_xd_fragment#?=&cb=fe17523f223b7&relation=parent.parent&transport=fragment&type=resize&height=20&ackdata"]

   # A common cause of runaway regex engines.
    - description: "Extract a URL with a ton of trailing periods"
      text: "Test a ton of periods http://example.com/path.........................................."
      expected: ["http://example.com/path"]

    - description: "Extract a URL with a ton of trailing commas"
      text: "Test a ton of periods http://example.com/,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"
      expected: ["http://example.com/"]

    - description: "Extract a URL with a ton of trailing '!'"
      text: "Test a ton of periods http://example.com/path/!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
      expected: ["http://example.com/path/"]

    - description: "DO NOT extract URLs in hashtag or @mention"
      text: "#test.com @test.com #http://test.com @http://test.com #t.co/abcde @t.co/abcde"
      expected: []

    - description: "extracts URLs in quote block"
      text: "test.com\n    other.com"
      expected: ["test.com", "other.com"]

    - description: "Extract a t.co URL with a trailing apostrophe"
      text: "I really like http://t.co/pbY2NfTZ's website"
      expected: ["http://t.co/pbY2NfTZ"]

    - description: "Extract a t.co URL with a trailing hyphen"
      text: "Check this site out http://t.co/FNkPfmii- it's great"
      expected: ["http://t.co/FNkPfmii"]

    - description: "Extract a t.co URL with a trailing colon"
      text: "According to http://t.co/ulYGBYSo: the internet is cool"
      expected: ["http://t.co/ulYGBYSo"]

    - description: "Extract URL before newline"
      text: "http://twitter.com\nhttp://example.com\nhttp://example.com/path\nexample.com/path\nt.co\nt.co/abcde"
      expected: ["http://twitter.com", "http://example.com", "http://example.com/path", "example.com/path", "t.co", "t.co/abcde"]

    - description: "DO NOT extract URL if preceded by $"
      text: "$http://twitter.com $twitter.com $http://t.co/abcde $t.co/abcde $t.co $TVI.CA $RBS.CA"
      expected: []

    - description: "Extract Plone view urls"
      text: "baar http://example.org/my-document/@@as-message foo"
      expected: ["http://example.org/my-document/@@as-message"]

    - description: "Extract url with 'corp' domain"
      text: "baar http://foobar.corp baz"
      expected: ["http://foobar.corp"]

    - description: "Extract url with 'corp' domain with path"
      text: "baar http://foobar.corp/enterprise foo"
      expected: ["http://foobar.corp/enterprise"]


  urls_with_indices:
    - description: "Extract a URL"
      text: "text http://google.com"
      expected:
        - url: "http://google.com"
          indices: [5, 22]

    - description: "Extract a URL inside a quote block"
      text: "    http://google.com    http://example.com"
      expected:
        - url: "http://google.com"
          indices: [4, 21]
        - url: "http://example.com"
          indices: [25, 43]

    - description: "Extract a URL from a Japanese tweet"
      text: "皆さん見てください！ http://google.com"
      expected:
        - url: "http://google.com"
          indices: [11, 28]

    - description: "Extract URLs without protocol on ccTLD with slash"
      text: "t.co/abcde bit.ly/abcde"
      expected:
        - url: "t.co/abcde"
          indices: [0, 10]
        - url: "bit.ly/abcde"
          indices: [11, 23]

    - description: "Extract URLs without protocol surrounded by CJK characters"
      text: "twitter.comこれは日本語です。example.com中国語t.co/abcde한국twitter.com example2.comテストtwitter.com/abcde"
      expected:
        - url: "twitter.com"
          indices: [0, 11]
        - url: "example.com"
          indices: [20, 31]
        - url: "t.co/abcde"
          indices: [34, 44]
        - url: "twitter.com"
          indices: [46, 57]
        - url: "example2.com"
          indices: [58, 70]
        - url: "twitter.com/abcde"
          indices: [73, 90]

    - description: "Extract URLs with and without protocol surrounded by CJK characters"
      text: "http://twitter.com/これは日本語です。example.com中国語http://t.co/abcde한국twitter.comテストexample2.comテストhttp://twitter.com/abcde"
      expected:
        - url: "http://twitter.com/"
          indices: [0, 19]
        - url: "example.com"
          indices: [28, 39]
        - url: "http://t.co/abcde"
          indices: [42, 59]
        - url: "twitter.com"
          indices: [61, 72]
        - url: "example2.com"
          indices: [75, 87]
        - url: "http://twitter.com/abcde"
          indices: [90, 114]

    - description: "Extract t.co URLs skipping trailing characters and adjusting indices correctly"
      text: "http://t.co/pbY2NfTZ's http://t.co/2vYHpAc5; http://t.co/ulYGBYSo: http://t.co/8MkmHU0k+c http://t.co/TKLp64dY.x http://t.co/8t7G3ddS#a http://t.co/FNkPfmii-"
      expected:
        - url: "http://t.co/pbY2NfTZ"
          indices: [0, 20]
        - url: "http://t.co/2vYHpAc5"
          indices: [23, 43]
        - url: "http://t.co/ulYGBYSo"
          indices: [45, 65]
        - url: "http://t.co/8MkmHU0k"
          indices: [67, 87]
        - url: "http://t.co/TKLp64dY"
          indices: [90, 110]
        - url: "http://t.co/8t7G3ddS"
          indices: [113, 133]
        - url: "http://t.co/FNkPfmii"
          indices: [136, 156]

    - description: "Extract correct indices for duplicate instances of the same URL"
      text: "http://t.co http://t.co"
      expected:
        - url: "http://t.co"
          indices: [0, 11]
        - url: "http://t.co"
          indices: [12, 23]

  hashtags:
    - description: "hashtag after anchor"
      text: "http://f.com/a#anchor #hashtag"
      expected: ["hashtag"]

    - description: "do not extract hashtag in quotes"
      text: "#got\n    #inquotes"
      expected: ["got"]

    - description: "hashtag after #"
      text: "aoeu#aoeu #tag"
      expected: ["tag"]

    - description: "Extract an all-alpha hashtag"
      text: "a #hashtag here"
      expected: ["hashtag"]

    - description: "Extract a letter-then-number hashtag"
      text: "this is #hashtag1"
      expected: ["hashtag1"]

    - description: "Extract a number-then-letter hashtag"
      text: "#1hashtag is this"
      expected: ["1hashtag"]

    - description: "Extract an all-numeric hashtag"
      text: "On the #16 bus"
      expected: ["16"]

    - description: "Extract a single numeric hashtag"
      text: "#0"
      expected: ["0"]

    - description: "Extract hashtag containing a dash"
      text: "hello #foo-bar"
      expected: ["foo-bar"]

    - description: "Extract hashtag after bracket"
      text: "(#hashtag1 )#hashtag2 [#hashtag3 ]#hashtag4 ’#hashtag5’#hashtag6"
      expected: ["hashtag1", "hashtag2", "hashtag3", "hashtag4", "hashtag5", "hashtag6"]

    - description: "Extract a hashtag containing ñ"
      text: "I'll write more tests #mañana"
      expected: ["mañana"]

    - description: "Extract a hashtag containing é"
      text: "Working remotely #café"
      expected: ["café"]

    - description: "Extract a hashtag containing ü"
      text: "Getting my Oktoberfest on #münchen"
      expected: ["münchen"]

    - description: "Extract a hashtag containing ğ"
      text: "in Turkish #dağ is a mountain"
      expected: ["dağ"]

    - description: "DO NOT Extract a hashtag containing Japanese"
      text: "this is not valid: # 会議中 ハッシュ"
      expected: []

    - description: "Extract a hashtag in Korean"
      text: "What is #트위터 anyway?"
      expected: ["트위터"]

    - description: "Extract a half-width Hangul hashtag"
      text: "Just random half-width Hangul #ﾣﾦﾰ"
      expected: ["ﾣﾦﾰ"]

    - description: "Extract a hashtag in Russian"
      text: "What is #ашок anyway?"
      expected: ["ашок"]

    - description: "Extract a hashtag in Greek"
      text: "#ελληνικά is a hashtag"
      expected: ["ελληνικά"]

    - description: "Extract a starting katakana hashtag"
      text: "#カタカナ is a hashtag"
      expected: ["カタカナ"]

    - description: "Extract a starting hiragana hashtag"
      text: "#ひらがな FTW!"
      expected: ["ひらがな"]

    - description: "Extract a starting kanji hashtag"
      text: "#漢字 is the future"
      expected: ["漢字"]

    - description: "Extract a trailing katakana hashtag"
      text: "Hashtag #カタカナ"
      expected: ["カタカナ"]

    - description: "Extract a trailing hiragana hashtag"
      text: "Japanese hashtags #ひらがな"
      expected: ["ひらがな"]

    - description: "Extract a trailing kanji hashtag"
      text: "Study time #漢字"
      expected: ["漢字"]

    - description: "Extract a central katakana hashtag"
      text: "See my #カタカナ hashtag?"
      expected: ["カタカナ"]

    - description: "Extract a central hiragana hashtag"
      text: "Study #ひらがな for fun and profit"
      expected: ["ひらがな"]

    - description: "Extract a central kanji hashtag"
      text: "Some say #漢字 is the past. what do they know?"
      expected: ["漢字"]

    - description: "Extract a Kanji/Katakana mixed hashtag"
      text: "日本語ハッシュタグテスト #日本語ハッシュタグ"
      expected: ["日本語ハッシュタグ"]

    - description: "Extract a hashtag after a punctuation"
      text: "日本語ハッシュテスト。#日本語ハッシュタグ"
      expected: ["日本語ハッシュタグ"]

    - description: "DO NOT include a punctuation in a hashtag"
      text: "#日本語ハッシュタグ。"
      expected: ["日本語ハッシュタグ"]

    - description: "Extract a full-width Alnum hashtag"
      text: "全角英数字ハッシュタグ ＃ｈａｓｈｔａｇ１２３"
      expected: ["ｈａｓｈｔａｇ１２３"]

    - description: "DO NOT extract a hashtag without a preceding space"
      text: "日本語ハッシュタグ#日本語ハッシュタグ"
      expected: []

    - description: "Hashtag with chouon"
      text: "長音ハッシュタグ。#サッカー"
      expected: ["サッカー"]

    - description: "Hashtag with half-width chouon"
      text: "長音ハッシュタグ。#ｻｯｶｰ"
      expected: ["ｻｯｶｰ"]

    - description: "Hashtag with half-widh voiced sounds marks"
      text: "#ﾊｯｼｭﾀｸﾞ #ﾊﾟﾋﾟﾌﾟﾍﾟﾎﾟ"
      expected: ["ﾊｯｼｭﾀｸﾞ", "ﾊﾟﾋﾟﾌﾟﾍﾟﾎﾟ"]

    - description: "Hashtag with half-width # after full-width ！"
      text: "できましたよー！#日本語ハッシュタグ。"
      expected: ["日本語ハッシュタグ"]

    - description: "Hashtag with full-width ＃ after full-width ！"
      text: "できましたよー！＃日本語ハッシュタグ。"
      expected: ["日本語ハッシュタグ"]

    - description: "Hashtag with ideographic iteration mark"
      text: "#云々 #学問のすゝめ #いすゞ #各〻"
      expected: ["云々", "学問のすゝめ", "いすゞ", "各〻"]

    - description: "Hashtags with ş (U+015F)"
      text: "Here’s a test tweet for you: #Ateş #qrşt #ştu #ş"
      expected: ["Ateş", "qrşt", "ştu", "ş"]

    - description: "Hashtag before punctuations"
      text: "#hashtag: #hashtag; #hashtag, #hashtag. #hashtag! #hashtag?"
      expected: ["hashtag", "hashtag", "hashtag", "hashtag", "hashtag", "hashtag"]

    - description: "Hashtag after punctuations"
      text: ":#hashtag ;#hashtag ,#hashtag .#hashtag !#hashtag ?#hashtag"
      expected: ["hashtag", "hashtag", "hashtag", "hashtag", "hashtag", "hashtag"]

    - description: "Hashtag before newline"
      text: "#hashtag\ntest\n#hashtag2\ntest\n#hashtag3\n"
      expected: ["hashtag", "hashtag2", "hashtag3"]

    - description: "DO NOT extract hashtag when # is followed by URL"
      text: "#http://twitter.com #https://twitter.com https://twitter.com/#login"
      expected: []

  hashtags_with_indices:
    - description: "Extract a hastag at the start"
      text: "#hashtag here"
      expected:
        - tag: "hashtag"
          indices: [0, 8]

    - description: "Extract a hastag at the end"
      text: "test a #hashtag"
      expected:
        - tag: "hashtag"
          indices: [7, 15]

    - description: "Extract a hastag in the middle"
      text: "test a #hashtag in a string"
      expected:
        - tag: "hashtag"
          indices: [7, 15]

    - description: "Extract a hastag with a dash"
      text: "test a #hash-tag in a string"
      expected:
        - tag: "hash-tag"
          indices: [7, 16]

    - description: "Extract only a valid hashtag"
      text: "#123 a #hashtag in a string"
      expected:
        - tag: "123"
          indices: [0, 4]
        - tag: "hashtag"
          indices: [7, 15]

    - description: "Extract a hashtag in a string of multi-byte characters"
      text: "会議中 #hashtag 会議中"
      expected:
        - tag: "hashtag"
          indices: [4, 12]

    - description: "Extract multiple valid hashtags"
      text: "One #two three #four"
      expected:
        - tag: "two"
          indices: [4, 8]
        - tag: "four"
          indices: [15, 20]

    - description: "Extract a non-latin hashtag"
      text: "Hashtags in #русский!"
      expected:
        - tag: "русский"
          indices: [12, 20]

    - description: "Extract multiple non-latin hashtags"
      text: "Hashtags in #中文, #日本語, #한국말, and #русский! Try it out!"
      expected:
        - tag: "中文"
          indices: [12, 15]
        - tag: "日本語"
          indices: [17, 21]
        - tag: "한국말"
          indices: [23, 27]
        - tag: "русский"
          indices: [33, 41]
