url_rewriting.py

"""URL rewriting tools

This module is about url and entry path rewriting.

The global scheme is the following:

Entries are stored in the ZIM file using their decoded fully decoded path:
- The full path is the full url without the scheme, username, password, port, fragment
 (ie : "<host>/<path>(?<query_string)"). See documentation of the `normalize` function
 for more details.
- urldecoded: the path itself must not be urlencoded or it would conflict with ZIM
  specification and readers won't be able to retrieve it, some parts (e.g. querystring)
  might be absorbed by a web server, ...
  . This is valid : "foo/part with space/bar?key=value"
  . This is NOT valid : "foo/part%20with%20space/bar%3Fkey%3Dvalue"
- even having multiple ? in a ZIM path is valid
  . This is valid :
    "foo/part/file with ? and +?who=Chip&Dale&question=It there any + here?"
  . This is NOT valid :
    "foo/part/file with %3F and +?who=Chip%26Dale&quer=Is%20there%20any%20%2B%20here%3F"
- space in query string must be stored as ` `, not `%2B`, `%20` or `+`, the `+` in a ZIM
  path means a `%2B in web resource (HTML document, ...):
  . This is valid : "foo/part/file?question=Is there any + here?"
  . This is NOT valid : "foo/part/file?question%3DIs%20there%20any%20%2B%20here%3F"

On top of that, fuzzy rules are applied on the ZIM path:
For instance a path "https://www.youtube.com/youtubei/v1/foo/baz/things?key=value
&other_key=other_value&videoId=xxxx&yet_another_key=yet_another_value"
is transformed to "youtube.fuzzy.replayweb.page/youtubei/v1/foo/baz/things?videoId=xxxx"
by slightly simplifying the path and keeping only the usefull arguments in the
querystring.

When rewriting documents (HTML, CSS, JS, ...), every time we find a URI to rewrite we
start by resolving it into an absolute URL (based on the containing document absolute
URI), applying the transformation to compute the corresponding ZIM path and we
url-encode the whole ZIM path, so that readers will have one single blob to process,
url-decode and find corresponding ZIM entry. Only '/' separators are considered safe
and not url-encoded.
"""

import re
from dataclasses import dataclass
from pathlib import PurePosixPath
from typing import ClassVar
from urllib.parse import quote, unquote, urljoin, urlsplit

import idna

from zimscraperlib import logger
from zimscraperlib.rewriting.rules import FUZZY_RULES


@dataclass
class AdditionalRule:
    match: re.Pattern[str]
    replace: str


COMPILED_FUZZY_RULES = [
    AdditionalRule(match=re.compile(rule["pattern"]), replace=rule["replace"])
    for rule in FUZZY_RULES
]


class HttpUrl:
    """A utility class representing an HTTP url, usefull to pass this data around

    Includes a basic validation, ensuring that URL is encoded, scheme is provided.
    """

    def __init__(self, value: str) -> None:
        HttpUrl.check_validity(value)
        self._value = value

    def __eq__(self, __value: object) -> bool:
        return isinstance(__value, HttpUrl) and __value.value == self.value

    def __hash__(self) -> int:
        return self.value.__hash__()

    def __str__(self) -> str:
        return f"HttpUrl({self.value})"

    def __repr__(self) -> str:
        return f"HttpUrl({self.value})"  # pragma: no cover

    @property
    def value(self) -> str:
        return self._value

    @classmethod
    def check_validity(cls, value: str) -> None:
        parts = urlsplit(value)

        if parts.scheme.lower() not in ["http", "https"]:
            raise ValueError(
                f"Incorrect HttpUrl scheme in value: {value} {parts.scheme}"
            )

        if not parts.hostname:
            raise ValueError(f"Unsupported empty hostname in value: {value}")

        if parts.hostname.lower() not in value:
            raise ValueError(f"Unsupported upper-case chars in hostname : {value}")


class ZimPath:
    """A utility class representing a ZIM path, usefull to pass this data around

    Includes a basic validation, ensuring that path does start with scheme, hostname,...
    """

    def __init__(self, value: str) -> None:
        ZimPath.check_validity(value)
        self._value = value

    def __eq__(self, __value: object) -> bool:
        return isinstance(__value, ZimPath) and __value.value == self.value

    def __hash__(self) -> int:
        return self.value.__hash__()

    def __str__(self) -> str:
        return f"ZimPath({self.value})"

    def __repr__(self) -> str:
        return f"ZimPath({self.value})"  # pragma: no cover

    @property
    def value(self) -> str:
        return self._value

    @classmethod
    def check_validity(cls, value: str) -> None:
        parts = urlsplit(value)

        if parts.scheme:
            raise ValueError(f"Unexpected scheme in value: {value} {parts.scheme}")

        if parts.hostname:
            raise ValueError(f"Unexpected hostname in value: {value} {parts.hostname}")

        if parts.username:
            raise ValueError(f"Unexpected username in value: {value} {parts.username}")

        if parts.password:
            raise ValueError(f"Unexpected password in value: {value} {parts.password}")


@dataclass
class RewriteResult:
    absolute_url: str
    rewriten_url: str
    zim_path: ZimPath | None


class ArticleUrlRewriter:
    """
    Rewrite urls in article.

    This is typically used to rewrite urls found in an HTML document, but can be used
    beyong that usage.
    """

    additional_rules: ClassVar[list[AdditionalRule]] = COMPILED_FUZZY_RULES

    def __init__(
        self,
        *,
        article_url: HttpUrl,
        article_path: ZimPath | None = None,
        existing_zim_paths: set[ZimPath] | None = None,
        missing_zim_paths: set[ZimPath] | None = None,
    ):
        """
        Initialise the rewriter

        Args:
          article_url: URL where the original document was located, used to resolve
        relative URLS which will be passed
          existing_zim_paths: list of ZIM paths which are known to exist, useful if one
        wants to rewrite the URL to a local one only if item exists in the ZIM
          missing_zim_paths: list of ZIM paths which are known to already be missing
        from the existing_zim_paths ; usefull only in complement with this variable ;
        new missing entries will be added as URLs are normalized in this function
        """
        self.article_path = article_path or ArticleUrlRewriter.normalize(article_url)
        self.article_url = article_url
        self.existing_zim_paths = existing_zim_paths
        self.missing_zim_paths = missing_zim_paths

    def get_item_path(self, item_url: str, base_href: str | None) -> ZimPath:
        """Utility to transform an item URL into a ZimPath"""

        item_absolute_url = urljoin(
            urljoin(self.article_url.value, base_href), item_url
        )
        return ArticleUrlRewriter.normalize(HttpUrl(item_absolute_url))

    def __call__(
        self,
        item_url: str,
        base_href: str | None,
        *,
        rewrite_all_url: bool = True,
    ) -> RewriteResult:
        """Rewrite a url contained in a article.

        The url is "fully" rewrited to point to a normalized entry path
        """

        try:
            item_url = item_url.strip()

            item_absolute_url = urljoin(
                urljoin(self.article_url.value, base_href), item_url
            )

            # Make case of standalone fragments more straightforward
            if item_url.startswith("#"):
                return RewriteResult(
                    absolute_url=item_absolute_url,
                    rewriten_url=item_url,
                    zim_path=None,
                )

            item_scheme = urlsplit(item_url).scheme
            if item_scheme and item_scheme not in ("http", "https"):
                return RewriteResult(
                    absolute_url=item_absolute_url,
                    rewriten_url=item_url,
                    zim_path=None,
                )

            item_fragment = urlsplit(item_absolute_url).fragment

            item_path = ArticleUrlRewriter.normalize(HttpUrl(item_absolute_url))

            if rewrite_all_url or (
                self.existing_zim_paths and item_path in self.existing_zim_paths
            ):
                return RewriteResult(
                    absolute_url=item_absolute_url,
                    rewriten_url=self.get_document_uri(item_path, item_fragment),
                    zim_path=item_path,
                )
            else:
                if (
                    self.missing_zim_paths is not None
                    and item_path not in self.missing_zim_paths
                ):
                    logger.debug(f"WARNING {item_path} ({item_url}) not in archive.")
                    # maintain a collection of missing Zim Path to not fill the logs
                    # with duplicate messages
                    self.missing_zim_paths.add(item_path)
                # The url doesn't point to a known entry
                return RewriteResult(
                    absolute_url=item_absolute_url,
                    rewriten_url=item_absolute_url,
                    zim_path=item_path,
                )

        except Exception as exc:  # pragma: no cover
            item_scheme = (
                item_scheme  # pyright: ignore[reportPossiblyUnboundVariable]
                if "item_scheme" in locals()
                else "<not_set>"
            )
            item_absolute_url = (
                item_absolute_url  # pyright: ignore[reportPossiblyUnboundVariable]
                if "item_absolute_url" in locals()
                else "<not_set>"
            )
            item_fragment = (
                item_fragment  # pyright: ignore[reportPossiblyUnboundVariable]
                if "item_fragment" in locals()
                else "<not_set>"
            )
            item_path = (
                item_path  # pyright: ignore[reportPossiblyUnboundVariable]
                if "item_path" in locals()
                else "<not_set>"
            )
            logger.debug(
                f"Invalid URL value found in {self.article_url.value}, kept as-is. "
                f"(item_url: {item_url}, "
                f"item_scheme: {item_scheme}, "
                f"item_absolute_url: {item_absolute_url}, "
                f"item_fragment: {item_fragment}, "
                f"item_path: {item_path}, "
                f"rewrite_all_url: {rewrite_all_url}",
                exc_info=exc,
            )
            return RewriteResult(
                absolute_url=item_absolute_url,
                rewriten_url=item_url,
                zim_path=None,
            )

    def get_document_uri(self, item_path: ZimPath, item_fragment: str) -> str:
        """Given an ZIM item path and its fragment, get the URI to use in document

        This function transforms the path of a ZIM item we want to adress from current
        document (HTML / JS / ...) and returns the corresponding URI to use.

        It computes the relative path based on current document location and escape
        everything which needs to be to transform the ZIM path into a valid RFC 3986 URI

        It also append a potential trailing item fragment at the end of the resulting
        URI.

        """
        item_parts = urlsplit(item_path.value)
        article_parts = urlsplit(self.article_path.value)

        # The relative path is computed using only the path components. A querystring is
        # part of the ZIM entry name (a leaf), it is not a navigable directory, so its
        # content (which may contain `/` or even `..` segments) must not be interpreted
        # as path navigation when computing the relative path ; doing so crashes on `..`
        # segments (see https://github.com/openzim/warc2zim/issues/380). The querystring
        # is re-appended and url-encoded together with the path below so that readers
        # consider path + querystring as a single whole entry.
        relative_path = str(
            PurePosixPath(item_parts.path).relative_to(
                (
                    PurePosixPath(article_parts.path)
                    if article_parts.path.endswith("/")
                    else PurePosixPath(article_parts.path).parent
                ),
                walk_up=True,
            )
        )
        # relative_to removes a potential last '/' in the path, we add it back before
        # appending the querystring
        if item_parts.path.endswith("/"):
            relative_path += "/"
        # re-append the querystring (part of the ZIM entry name) now that the relative
        # path has been computed
        if item_parts.query:
            relative_path += "?" + item_parts.query

        return (
            f"{quote(relative_path, safe='/')}"
            f"{'#' + item_fragment if item_fragment else ''}"
        )

    @classmethod
    def apply_additional_rules(cls, uri: HttpUrl | str) -> str:
        """Apply additional rules on a URL or relative path

        First matching additional rule matching the input value is applied and its
        result is returned.

        If no additional rule is matching, the input is returned as-is.
        """
        value = uri.value if isinstance(uri, HttpUrl) else uri
        for rule in cls.additional_rules:
            if match := rule.match.match(value):
                return match.expand(rule.replace)
        return value

    @classmethod
    def normalize(cls, url: HttpUrl) -> ZimPath:
        """Transform a HTTP URL into a ZIM path to use as a entry's key.

        According to RFC 3986, a URL allows only a very limited set of characters, so we
        assume by default that the url is encoded to match this specification.

        The transformation rewrites the hostname, the path and the querystring.

        The transformation drops the URL scheme, username, password, port and fragment:
        - we suppose there is no conflict of URL scheme or port: there is no two
        ressources with same hostname, path and querystring but different URL scheme or
          port leading to different content
        - we consider username/password port are purely authentication mechanism which
        have no impact on the content to server
        - we know that the fragment is never passed to the server, it stays in the
        User-Agent, so if we encounter a fragment while normalizing a URL found in a
        document, it won't make its way to the ZIM anyway and will stay client-side

        The transformation consists mainly in decoding the three components so that ZIM
        path is not encoded at all, as required by the ZIM specification.

        Decoding is done differently for the hostname (decoded with puny encoding) and
        the path and querystring (both decoded with url decoding).

        The final transformation is the application of fuzzy rules (sourced from wabac)
        to transform some URLs into replay URLs and drop some useless stuff.

        Returned value is a ZIM path, without any puny/url encoding applied, ready to be
        passed to python-libzim for UTF-8 encoding.
        """

        url_parts = urlsplit(url.value)

        if not url_parts.hostname:
            # cannot happen because of the HttpUrl checks, but important to please the
            # type checker
            raise Exception("Hostname is missing")  # pragma: no cover

        # decode the hostname if it is punny-encoded
        hostname = (
            idna.decode(url_parts.hostname)
            if url_parts.hostname.startswith("xn--")
            else url_parts.hostname
        )

        path = url_parts.path

        if path:
            # unquote the path so that it is stored unencoded in the ZIM as required by
            # ZIM specification
            path = unquote(path)
        else:
            # if path is empty, we need a "/" to remove ambiguities, e.g.
            # https://example.com and https://example.com/ must all lead to the same ZIM
            # entry to match RFC 3986 section 6.2.3:
            # https://www.rfc-editor.org/rfc/rfc3986#section-6.2.3
            path = "/"

        query = url_parts.query

        # if query is missing, we do not add it at all, not even a trailing ? without
        # anything after it
        if url_parts.query:
            # `+`` in query parameter must be decoded as space first to remove
            # ambiguities between a space (encoded as `+` in url query parameter) and a
            # real plus sign (encoded as %2B but soon decoded in ZIM path)
            query = query.replace("+", " ")
            # unquote the query so that it is stored unencoded in the ZIM as required by
            # ZIM specification
            query = "?" + unquote(query)
        else:
            query = ""

        fuzzified_url = ArticleUrlRewriter.apply_additional_rules(
            f"{hostname}{ArticleUrlRewriter._remove_subsequent_slashes(path)}{ArticleUrlRewriter._remove_subsequent_slashes(query)}"
        )

        return ZimPath(fuzzified_url)

    @classmethod
    def _remove_subsequent_slashes(cls, value: str) -> str:
        """Remove all successive occurence of a slash `/` in a given string

        E.g `val//ue` or `val///ue` or `val////ue` (and so on) are transformed into
        `value`
        """
        return re.sub(r"//+", "/", value)