euscan-ng/bin/euscan_patch_metadata

#!/usr/bin/env python

import os
import sys
import re
import urllib
from tempfile import mkstemp
import tarfile
import gzip
import logging
import shutil
from difflib import unified_diff

from portage.exception import AmbiguousPackageName
from gentoolkit.query import Query
from BeautifulSoup import BeautifulSoup, SoupStrainer


logger = logging.getLogger(__name__)


# From portage-janitor
def guess_indent_values(before):
    rindent = -1
    indent = -1
    tab = False

    def guess_for_tags(tags):
        for tag in tags:
            for i in [0, 2, 4, 6, 8, 12, 16]:
                if '\n%s<%s' % (' ' * i, tag) in before:
                    return i, False
            for i in [0, 1, 2]:
                if '\n%s<%s' % ('\t' * i, tag) in before:
                    return i, True
        return -1, False

    rindent, tab = guess_for_tags(
        ['herd', 'maintainer', 'longdescription', 'use', 'upstream']
    )
    if rindent == -1:
        rindent = 2
    rindent_str = ('\t' if tab else ' ') * rindent
    indent, tab = guess_for_tags(['watch', 'name', 'email'])
    if indent == -1:
        indent = rindent * 2 if rindent else 4
        if rindent and rindent_str == '\t':
            tab = True
    indent_str = ('\t' if tab else ' ') * indent
    return rindent_str, indent_str


def get_watch_data(package):
    deb_url, deb_type = get_deb_url(package.name)
    if deb_type == "source":
        return handle_source(deb_url)
    if deb_type == "diff":
        return handle_diff(deb_url)


def handle_diff(deb_url):
    _, temp_deb = mkstemp()

    logger.info("  Downloading debian diff %s...", deb_url)
    urllib.urlretrieve(deb_url, temp_deb)

    watch_data = ""

    fp = gzip.open(temp_deb, 'rb')
    for line in fp:
        if re.match(r"\+\+\+ .+?/debian/watch", line):
            fp.readline()  # diff lines, don't care
            cur_line = fp.readline()
            while cur_line.startswith("+"):
                watch_data += cur_line[1:]
                cur_line = fp.readline()
    fp.close()

    os.unlink(temp_deb)

    return watch_data


def handle_source(deb_url):
    _, temp_deb = mkstemp()
    temp_dir = os.path.dirname(temp_deb)

    logger.info("  Downloading debian source %s...", deb_url)
    urllib.urlretrieve(deb_url, temp_deb)
    tar = tarfile.open(temp_deb)

    watch_data = None

    try:
        tar.extract("debian/watch", temp_dir)
    except KeyError:
        pass
    else:
        debian_path = os.path.join(temp_dir, "debian")
        watch_path = os.path.join(debian_path, "watch")
        watch_data = open(os.path.join(watch_path)).read()
        shutil.rmtree(debian_path)

    os.unlink(temp_deb)

    return watch_data


def get_deb_url(name):
    deb_url = None
    deb_type = None

    while not deb_url:
        url = "http://packages.debian.org/source/unstable/%s" % name
        opened = urllib.urlopen(url)

        content = opened.read()

        for link in BeautifulSoup(content, parseOnlyThese=SoupStrainer("a")):
            if re.match("[^\s]+\.debian\.tar\.(?:gz|bz2)", link.text):
                deb_url = link["href"]
                deb_type = "source"
                break
            if re.match("[^\s]+\.diff\.gz", link.text):
                deb_url = link["href"]
                deb_type = "diff"
                break

        if not deb_url:
            logger.error("  Cannot get package from %s" % url)
            name = raw_input("  Package name in Debian: ")

    return deb_url, deb_type


def patch_metadata(package, watch_data, diff=False):
    logger.info("  Patching metadata file")

    metadata_path = package.metadata.metadata_path

    with open(metadata_path) as fp:
        original = fp.read()
    rindent, indent = guess_indent_values(original)
    data = original

    # clean watch_data
    watch_data = "\n".join([line for line in watch_data.split("\n")
                            if not line.startswith("#")])  # comments

    watch_data = watch_data.replace("\\\n", "")  # remove backslashes

    watch_tags = []

    for watch_line in watch_data.split("\n"):  # there can be multiple lines
        watch_line = " ".join(watch_line.split())  # remove extra spaces and \n

        version_parse = re.match("version=(\d+?)", watch_line)
        if version_parse:
            version = version_parse.group(1)
            continue

        if not watch_line:  # skip empty lines
            continue

        # parse watch_line
        result = re.match(
            r'(?:opts=(?:"([^"]+?)"|([^\s]+?)) )?(.*)',
            watch_line
        )

        opts_quote, opts, url = result.groups()
        opts = opts_quote or opts

        if opts:
            # clean opts, skip useless ones
            valid = ("uversionmangle", "versionmangle", "downloadurlmangle")
            cleaned_opts = []
            for opt in opts.split(","):
                opt_name, opt_value = opt.split("=", 1)
                if opt_name in valid:
                    if opt_name == "uversionmangle": opt_name = "versionmangle"
                    cleaned_opts.append('%s="%s"' % (opt_name, opt_value))
            opts = " ".join(cleaned_opts)

        # clean url from useless stuff. Just keep <base> [<filepattern>]
        url_search = re.search(r"^([^\s]+)(?: ([^\s]*\([^\s]+\)[^\s]*))?", url)
        url = " ".join([x for x in url_search.groups() if x is not None])

        if opts:
            watch_tag = '%s<watch version="%s" %s>%s</watch>' % \
                (indent, version, opts, url)
        else:
            watch_tag = '%s<watch version="%s">%s</watch>' % \
                (indent, version, url)
        watch_tags.append(watch_tag)

    watch_tags = "\n".join(watch_tags)

    if '<upstream>' in data:
        data = data.replace('<upstream>', '<upstream>\n%s' % watch_tags, 1)
    else:
        rep = '%s<upstream>\n%s\n%s</upstream>\n</pkgmetadata>' % \
            (rindent, watch_tags, rindent)
        data = data.replace('</pkgmetadata>', rep, 1)

    if not diff:
        return data
    else:
        # Generate clean a/category/package/metadata.xml path
        n = metadata_path.find(package.category)
        if n != -1:
            metadata_path = metadata_path[n:]
        res = unified_diff(
            original.splitlines(True),
            data.splitlines(True),
            fromfile=os.path.join('a/', metadata_path),
            tofile=os.path.join('b/', metadata_path),
        )
        return "".join([x for x in res])


def process_package(query, diff=False):
    try:
        matches = Query(query).smart_find(
                in_installed=True,
                in_porttree=True,
                in_overlay=True,
                include_masked=True,
                show_progress=False,
                no_matches_fatal=False,
        )
    except AmbiguousPackageName:
        logger.error("  Ambiguous package name")
        return None

    if len(matches) == 0:
        logger.error("  Package not found")
        return None

    matches = sorted(matches)
    package = matches.pop()
    if '9999' in package.version and len(matches) > 0:
        package = matches.pop()

    watch_data = get_watch_data(package)
    if watch_data is None:
        logger.error("  No watch file found")
    else:
        return patch_metadata(package, watch_data, diff=diff)


def main():
    import optparse
    p = optparse.OptionParser(
        usage="usage: %prog <package> [<package> [...]]",
    )
    p.add_option('-d', '--diff', action="store_true", dest="diff",
                 default=False,
                 help="Outputs a diff")
    opts, packages = p.parse_args()

    logging.basicConfig(stream=sys.stderr, level=logging.INFO,
                        format='%(message)s')

    for package in packages:
        logger.info("Processing %s..." % package)
        result = process_package(package, opts.diff)
        if result:
            sys.stdout.write(result)

if __name__ == "__main__":
    main()