!"}
diff --git a/tests/test_hyperlinks.py b/tests/test_hyperlinks.py
new file mode 100644
index 0000000..5fb1e63
--- /dev/null
+++ b/tests/test_hyperlinks.py
@@ -0,0 +1,64 @@
+import gumbocy
+from test_word_groups import TAGS_SEPARATORS
+
+
+def _links(html, url=None):
+ parser = gumbocy.HTMLParser(options={
+ "tags_separators": TAGS_SEPARATORS
+ })
+ parser.parse(html)
+ ret = parser.analyze()
+ return ret["internal_hyperlinks"] + ret["external_hyperlinks"]
+
+
+def test_get_hyperlinks():
+ links = _links("""Test titlex""")
+ assert len(links) == 0
+
+ links = _links("""Test title
+ Y
+ """)
+ assert len(links) == 0
+
+ links = _links("""Test title
+ Y
+ """)
+ assert len(links) == 0
+
+ links = _links("""Test title
+ Y
+ """)
+ assert len(links) == 0
+
+ links = _links("""Test title
+ Y
+ """)
+ assert len(links) == 0
+
+ links = _links("""Test title
+ Y
+ """)
+ assert len(links) == 0
+
+ links = _links("""Test title
+ Y
+ """)
+ assert len(links) == 1
+ assert links[0][0] == "http://sub.test.com/page1?q=2&a=b#xxx"
+ assert links[0][1] == "Y"
+
+ links = _links("""Test title
+ Y X
+ """, url="http://sub.test.com/page2")
+ assert len(links) == 1
+ assert links[0][0] == "/page1?q=2&a=b#xxx"
+ assert links[0][1] == "Y X"
+
+ links = _links("""Test title
+ Y Z
+ """, url="http://sub.test.com/page2/x.html")
+ assert len(links) == 1
+ assert links[0][0] == "../page1?q=2&a=b#xxx"
+ assert links[0][1] == "Y Z"
+
+ # TODO resolution tests
\ No newline at end of file
diff --git a/tests/test_listnodes.py b/tests/test_listnodes.py
index 7ecc0f3..05d33f0 100644
--- a/tests/test_listnodes.py
+++ b/tests/test_listnodes.py
@@ -2,9 +2,9 @@
def listnodes(html, options=None):
- parser = gumbocy.HTMLParser(html)
- parser.parse()
- return parser.listnodes(options=options)
+ parser = gumbocy.HTMLParser(options=options)
+ parser.parse(html)
+ return parser.listnodes()
def test_basic():
diff --git a/tests/test_word_groups.py b/tests/test_word_groups.py
new file mode 100644
index 0000000..ffd218b
--- /dev/null
+++ b/tests/test_word_groups.py
@@ -0,0 +1,134 @@
+import gumbocy
+import pytest
+
+TAGS_SEPARATORS = frozenset([
+ "body",
+
+ # http://www.w3.org/TR/html5/grouping-content.html#grouping-content
+ "p", "pre", "blockquote", "ul", "ol", "li", "dl", "dt", "dd", "figure", "figcaption",
+
+ "br", "img",
+
+ "h1", "h2", "h3", "h4", "h5", "h6"
+])
+
+
+SAMPLES = [
+ {
+ "html": """ hello
""",
+ "groups": [
+ ("hello", "p")
+ ]
+ },
+
+ # A is automatically added
+ {
+ "html": """ nobody """,
+ "groups": [
+ ("nobody", "body")
+ ]
+ },
+
+ # span
+ {
+ "html": """ pre link post
""",
+ "groups": [
+ ("pre link post", "p")
+ ]
+ },
+
+ # a
+ {
+ "html": """ pre link post
""",
+ "groups": [
+ ("pre link post", "p")
+ ]
+ },
+
+ # mid p
+ {
+ "html": """ pre
mid post
""",
+ "groups": [
+ ("pre", "p"),
+ ("li1 x", "li"),
+ ("mid", "body"),
+ ("post", "p")
+ ]
+ },
+
+ # Lists
+ {
+ "html": """ pre post """,
+ "groups": [
+ ("pre", "body"),
+ ("li1", "li"),
+ ("li2", "li"),
+ ("post", "body")
+ ]
+ },
+
+ # HR with illegal . "post" is actually part of
.
+ {
+ "html": """ pre
post""",
+ "groups": [
+ ("pre", "p"),
+ ("post", "body")
+ ]
+ },
+
+ # Non-closed p tag.
+ {
+ "html": """ pre post""",
+ "groups": [
+ ("pre", "body"),
+ ("post", "p")
+ ]
+ },
+
+ # BR
+ {
+ "html": """
pre
post
""",
+ "groups": [
+ ("pre", "p"),
+ ("post", "p")
+ ]
+ },
+
+ # IMG filename + alt
+ {
+ "html": """ pre 
post
""",
+ "groups": [
+ ("pre", "p"),
+ ("james brown", "img"),
+ # ("maceo parker", "img"),
+ ("post", "p")
+ ]
+ },
+
+ # IMG with dataURIs are ignored
+ {
+ "html": """ pre 
post
""",
+ "groups": [
+ ("pre", "p"),
+ ("Red dot", "img"),
+ ("post", "p")
+ ]
+ },
+]
+
+
+# TODO: good coverage of http://www.w3.org/html/wg/drafts/html/master/syntax.html
+@pytest.mark.parametrize(("sample"), SAMPLES)
+def test_get_word_groups(sample):
+
+ parser = gumbocy.HTMLParser(options={
+ "tags_separators": TAGS_SEPARATORS,
+ "attributes_whitelist": ["src", "alt"]
+ })
+ parser.parse(sample["html"])
+ parsed = parser.analyze()
+
+ for i, group in enumerate(parsed["word_groups"]):
+ assert group == sample["groups"][i]
+
+ assert len(parsed["word_groups"]) == len(sample["groups"])