euscan-ng/bin/euscan_patch_metadata

#!/usr/bin/env python

import os
import sys
import re
import urllib
from tempfile import mkstemp
import tarfile
import gzip
import logging
import shutil
from difflib import unified_diff

from portage.exception import AmbiguousPackageName
from gentoolkit.query import Query
from BeautifulSoup import BeautifulSoup, SoupStrainer


logger = logging.getLogger(__name__)


# From portage-janitor
def guess_indent_values(before):
    rindent = -1
    indent = -1
    tab = False

    def guess_for_tags(tags):
        for tag in tags:
            for i in [0, 2, 4, 6, 8, 12, 16]:
                if "\n%s<%s" % (" " * i, tag) in before:
                    return i, False
            for i in [0, 1, 2]:
                if "\n%s<%s" % ("\t" * i, tag) in before:
                    return i, True
        return -1, False

    rindent, tab = guess_for_tags(
        ["herd", "maintainer", "longdescription", "use", "upstream"]
    )
    if rindent == -1:
        rindent = 2
    rindent_str = ("\t" if tab else " ") * rindent
    indent, tab = guess_for_tags(["watch", "name", "email"])
    if indent == -1:
        indent = rindent * 2 if rindent else 4
        if rindent and rindent_str == "\t":
            tab = True
    indent_str = ("\t" if tab else " ") * indent
    return rindent_str, indent_str


def get_watch_data(package):
    deb_url, deb_type = get_deb_url(package.name)
    if deb_type == "source":
        return handle_source(deb_url)
    if deb_type == "diff":
        return handle_diff(deb_url)


def handle_diff(deb_url):
    _, temp_deb = mkstemp()

    logger.info("  Downloading debian diff %s...", deb_url)
    urllib.urlretrieve(deb_url, temp_deb)

    watch_data = ""

    fp = gzip.open(temp_deb, "rb")
    for line in fp:
        if re.match(r"\+\+\+ .+?/debian/watch", line):
            fp.readline()  # diff lines, don't care
            cur_line = fp.readline()
            while cur_line.startswith("+"):
                watch_data += cur_line[1:]
                cur_line = fp.readline()
    fp.close()

    os.unlink(temp_deb)

    return watch_data


def handle_source(deb_url):
    _, temp_deb = mkstemp()
    temp_dir = os.path.dirname(temp_deb)

    logger.info("  Downloading debian source %s...", deb_url)
    urllib.urlretrieve(deb_url, temp_deb)
    tar = tarfile.open(temp_deb)

    watch_data = None

    try:
        tar.extract("debian/watch", temp_dir)
    except KeyError:
        pass
    else:
        debian_path = os.path.join(temp_dir, "debian")
        watch_path = os.path.join(debian_path, "watch")
        watch_data = open(os.path.join(watch_path)).read()
        shutil.rmtree(debian_path)

    os.unlink(temp_deb)

    return watch_data


def get_deb_url(name):
    deb_url = None
    deb_type = None

    while not deb_url:
        url = "http://packages.debian.org/source/unstable/%s" % name
        opened = urllib.urlopen(url)

        content = opened.read()

        for link in BeautifulSoup(content, parseOnlyThese=SoupStrainer("a")):
            if re.match("[^\s]+\.debian\.tar\.(?:gz|bz2)", link.text):
                deb_url = link["href"]
                deb_type = "source"
                break
            if re.match("[^\s]+\.diff\.gz", link.text):
                deb_url = link["href"]
                deb_type = "diff"
                break

        if not deb_url:
            logger.error("  Cannot get package from %s" % url)
            name = raw_input("  Package name in Debian: ")

    return deb_url, deb_type


def patch_metadata(package, watch_data, diff=False):
    logger.info("  Patching metadata file")

    metadata_path = package.metadata.metadata_path

    with open(metadata_path) as fp:
        original = fp.read()
    rindent, indent = guess_indent_values(original)
    data = original

    # clean watch_data
    watch_data = "\n".join(
        [line for line in watch_data.split("\n") if not line.startswith("#")]
    )  # comments

    watch_data = watch_data.replace("\\\n", "")  # remove backslashes

    watch_tags = []

    for watch_line in watch_data.split("\n"):  # there can be multiple lines
        watch_line = " ".join(watch_line.split())  # remove extra spaces and \n

        version_parse = re.match("version=(\d+?)", watch_line)
        if version_parse:
            version = version_parse.group(1)
            continue

        if not watch_line:  # skip empty lines
            continue

        # parse watch_line
        result = re.match(r'(?:opts=(?:"([^"]+?)"|([^\s]+?)) )?(.*)', watch_line)

        opts_quote, opts, url = result.groups()
        opts = opts_quote or opts

        if opts:
            # clean opts, skip useless ones
            valid = ("uversionmangle", "versionmangle", "downloadurlmangle")
            cleaned_opts = []
            for opt in opts.split(","):
                opt_name, opt_value = opt.split("=", 1)
                if opt_name in valid:
                    if opt_name == "uversionmangle":
                        opt_name = "versionmangle"
                    cleaned_opts.append('%s="%s"' % (opt_name, opt_value))
            opts = " ".join(cleaned_opts)

        # clean url from useless stuff. Just keep <base> [<filepattern>]
        url_search = re.search(r"^([^\s]+)(?: ([^\s]*\([^\s]+\)[^\s]*))?", url)
        url = " ".join([x for x in url_search.groups() if x is not None])

        if opts:
            watch_tag = '%s<watch version="%s" %s>%s</watch>' % (
                indent,
                version,
                opts,
                url,
            )
        else:
            watch_tag = '%s<watch version="%s">%s</watch>' % (indent, version, url)
        watch_tags.append(watch_tag)

    watch_tags = "\n".join(watch_tags)

    if "<upstream>" in data:
        data = data.replace("<upstream>", "<upstream>\n%s" % watch_tags, 1)
    else:
        rep = "%s<upstream>\n%s\n%s</upstream>\n</pkgmetadata>" % (
            rindent,
            watch_tags,
            rindent,
        )
        data = data.replace("</pkgmetadata>", rep, 1)

    if not diff:
        return data
    else:
        # Generate clean a/category/package/metadata.xml path
        n = metadata_path.find(package.category)
        if n != -1:
            metadata_path = metadata_path[n:]
        res = unified_diff(
            original.splitlines(True),
            data.splitlines(True),
            fromfile=os.path.join("a/", metadata_path),
            tofile=os.path.join("b/", metadata_path),
        )
        return "".join([x for x in res])


def process_package(query, diff=False):
    try:
        matches = Query(query).smart_find(
            in_installed=True,
            in_porttree=True,
            in_overlay=True,
            include_masked=True,
            show_progress=False,
            no_matches_fatal=False,
        )
    except AmbiguousPackageName:
        logger.error("  Ambiguous package name")
        return None

    if len(matches) == 0:
        logger.error("  Package not found")
        return None

    matches = sorted(matches)
    package = matches.pop()
    if "9999" in package.version and len(matches) > 0:
        package = matches.pop()

    watch_data = get_watch_data(package)
    if watch_data is None:
        logger.error("  No watch file found")
    else:
        return patch_metadata(package, watch_data, diff=diff)


def main():
    import optparse

    p = optparse.OptionParser(
        usage="usage: %prog <package> [<package> [...]]",
    )
    p.add_option(
        "-d",
        "--diff",
        action="store_true",
        dest="diff",
        default=False,
        help="Outputs a diff",
    )
    opts, packages = p.parse_args()

    logging.basicConfig(stream=sys.stderr, level=logging.INFO, format="%(message)s")

    for package in packages:
        logger.info("Processing %s..." % package)
        result = process_package(package, opts.diff)
        if result:
            sys.stdout.write(result)


if __name__ == "__main__":
    main()