cosr-back/cosrlib/url.py at master · commonsearch/cosr-back · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
# pylint: disable=too-many-branches

from __future__ import absolute_import, division, print_function, unicode_literals

import urllib
# import tldextract as _tldextract
import urlparse4 as urlparse

from pyfaup.faup import Faup

from . import py2_unicode


def tld_extract(domain):

    if "_faup" not in __builtins__:
        __builtins__["_faup"] = Faup()
    _faup = __builtins__["_faup"]
    _faup.decode(domain.decode("utf-8").strip(b"."))
    return (_faup.get_subdomain() or b"", _faup.get_domain_without_tld() or b"", _faup.get_tld() or b"")

# TODO init lazily
# _tldextractor = _tldextract.TLDExtract(suffix_list_urls=None)


class URL(object):
    """ Base class for manipulating an URL without context """

    def __init__(self, url, check_encoding=False):

        if isinstance(url, py2_unicode):
            self.url = url.encode("utf-8")
        else:
            self.url = url

        if check_encoding:
            try:
                self.url.decode('ascii')
            except UnicodeDecodeError:
                p = urlparse.urlsplit(self.url)

                # TODO: check the rightfulness of this!
                self.url = urlparse.urlunsplit((
                    p[0],
                    p[1],
                    urllib.quote(p[2], safe=b"/"),
                    urllib.quote(p[3], safe=b"&?="),
                    urllib.quote(p[4])
                ))

    def urljoin(self, href):
        """ Optimized version of urlparse.urljoin() """
        return urlparse.urljoin(self.url, href)

    # Allow picking/unpickling
    def __getstate__(self):
        return self.url

    def __setstate__(self, state):
        self.url = state

    # This is only called when the attribute is still missing
    def __getattr__(self, attr):
        # pylint: disable=redefined-variable-type

        if attr == "parsed":
            # try:
            value = urlparse.urlsplit(self.url)
            # except ValueError:
            #     value = urlparse.urlsplit("about:blank")

        elif attr == "tldextracted":

            value = tld_extract(self.parsed.netloc)
            # value = _tldextractor(self.url)

        elif attr == "normalized":
            value = urlparse.urlunsplit((
                None,
                self.normalized_domain,
                self.parsed.path if self.parsed.path else b"/",
                self.parsed.query,
                b""
            )).lstrip(b"/")

            if value.count(b"/") == 1:
                value = value.strip(b"/")

        elif attr == "normalized_without_query":
            value = urlparse.urlunsplit((
                None,
                self.normalized_domain,
                self.parsed.path if self.parsed.path else b"/",
                b"",
                b""
            )).lstrip(b"/")

            if value.count(b"/") == 1:
                value = value.strip(b"/")

        elif attr == "homepage":
            value = urlparse.urlunsplit((
                self.parsed.scheme,
                self.domain,
                b"/",
                b"",
                b""
            )).strip(b"/")

        # Pay-level domain
        elif attr == "pld":
            value = b"%s.%s" % (self.tldextracted[1], self.tldextracted[2])

        elif attr == "domain":
            value = self.parsed.netloc

        elif attr == "subdomain":
            value = self.tldextracted[0]

        elif attr == "normalized_domain":

            value = self.domain.strip(b".")

            while value.startswith(b"www."):
                value = value[4:]

            if value.endswith(b':80'):
                value = value[:-3]
            elif value.endswith(b':443'):
                value = value[:-4]

            value = value.strip(b".")

        elif attr == "normalized_subdomain":

            value = self.subdomain.strip(b".")

            if value == b"www":
                value = b""
            else:
                while value.startswith(b"www."):
                    value = value[4:]

        elif attr == "normalized_path":
            if self.parsed.path == b"/":
                return b""
            return self.parsed.path

        # https://en.wikipedia.org/wiki/Public_Suffix_List
        # Returns the domain name suffix ("co.uk" for "bbc.co.uk")
        elif attr == "suffix":
            value = self.tldextracted[2]

        else:
            raise Exception("Unknown attribute %s !" % attr)

        self.__dict__[attr] = value
        return value