euscan: misc fixes

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>

TODO

@@ -22,6 +22,7 @@ euscan
 Site Handlers
 -------------
 
+- store un-expanded SRC_URI and use it in handlers
 - berlios
 - sourceforge: http://sourceforge.net/api/file/index/project-name/vboxgtk/mtime/desc/limit/20/rss  http://sourceforge.net/api/release/index/project-id/264534/rss
 - MySQL: should use http://downloads.mysql.com/archives/

bin/euscan

@@ -38,7 +38,6 @@ from euscan.out import progress_bar
 isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty()
 isatty_stderr = os.environ.get('TERM') != 'dumb' and sys.stderr.isatty()
 
-
 def exit_helper(status):
     if CONFIG["format"]:
         print(output.get_formatted_output())

euscanwww/djeuscan/processing/misc/update_portage_trees.py

@@ -53,6 +53,14 @@ def emerge_sync(logger):
            "--config-root", settings.PORTAGE_CONFIGROOT]
     return _launch_command(cmd, logger)
 
+def emerge_metadata(logger):
+    """
+    Launches an emerge --metadata
+    """
+    cmd = ["emerge", "--metadata", "--root", settings.PORTAGE_ROOT,
+           "--config-root", settings.PORTAGE_CONFIGROOT]
+    return _launch_command(cmd, logger)
+
 
 def layman_sync(logger, cache=True):
     """
@@ -106,7 +114,8 @@ def update_portage_trees(logger=None):
     logger.info("Running layman --sync")
     layman_sync(logger, cache=True)
     #logger.info("Running emerge --regen")
-    #emerge_regen()
+    #emerge_regen(logger)
+    emerge_metadata(logger)
     logger.info("Running eix-update")
     eix_update(logger)
     logger.info("Done!")

euscanwww/djeuscan/processing/scan/scan_metadata.py

@@ -91,8 +91,10 @@ class ScanMetadata(object):
         try:
             metadata = None
             pkg, metadata = self.metadata_from_db(query, pkg)
+
             if not metadata:
                 pkg, metadata = self.metadata_from_portage(query, pkg)
+
             if not metadata:
                 return
         except Exception as e:
@@ -148,6 +150,7 @@ class ScanMetadata(object):
                     self.style.ERROR("Bad maintainer: '%s' '%s'" % \
                                          (maintainer.name, maintainer.email))
                 )
+
         pkg.save()
 
     def store_herd(self, name, email):

euscanwww/djeuscan/tasks.py

@@ -223,7 +223,7 @@ def update_upstream():
 def scan_package(package):
     scan_portage([package], purge_packages=True, purge_versions=True)
     scan_metadata([package])
-    scan_upstream([package])
+    scan_upstream([package], purge_versions=True)
     return True
 
 
@@ -254,7 +254,7 @@ def consume_refresh_queue(locked=False):
         pkg = query.package
         query.delete()
         scan_package_user.delay(pkg)
-        logger.info('Done (%s)' % pkg)
+        logger.info('Selected: %s' % pkg)
     except IndexError:
         pass
     finally:

euscanwww/euscanwww/settings.py

@@ -222,7 +222,6 @@ ACCOUNT_ACTIVATION_DAYS = 7
 RECAPTCHA_PUBLIC_KEY = ""
 RECAPTCHA_PRIVATE_KEY = ""
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
-EMAIL_FROM = 'noreply@iksaif.net'
 
 # djeuscan tasks
 PORTAGE_ROOT = "/"

pym/euscan/handlers/berlios.py

@@ -12,7 +12,7 @@ CONFIDENCE = 90
 PRIORITY = 90
 
 
-berlios_regex = r"berlios.de/([^/]+)/([^/]+)"
+berlios_regex = r"mirror://berlios/([^/]+)/([^/]+)"
 
 
 def can_handle(pkg, url=None):

pym/euscan/handlers/google_code.py

@@ -32,6 +32,6 @@ def scan_url(pkg, url, options):
     result = url_scan(pkg, base_url, file_pattern)
 
     ret = []
-    for found_url, pv, _, _ in result:
+    for url, pv, _, _ in result:
-        ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE))
+        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
     return ret

pym/euscan/handlers/sourceforge.py

@@ -12,7 +12,7 @@ PRIORITY = 90
 
 
 def can_handle(pkg, url=None):
-    return url and "sourceforge.net/" in url
+    return url and "mirror://sourceforge/" in url
 
 
 def scan_url(pkg, url, options):
@@ -20,7 +20,7 @@ def scan_url(pkg, url, options):
 
     cp, ver, rev = portage.pkgsplit(pkg.cpv)
 
-    project, filename = re.search("sourceforge.net/([^/]+)/(?:.*/)?([^/]+)",
+    project, filename = re.search("mirror://sourceforge/([^/]+)/(?:.*/)?([^/]+)",
                                   url).groups()
     base_url = "http://qa.debian.org/watch/sf.php/%s" % project
     file_pattern = regex_from_template(
@@ -30,6 +30,6 @@ def scan_url(pkg, url, options):
     result = url_scan(pkg, base_url, file_pattern)
 
     ret = []
-    for found_url, pv, _, _ in result:
+    for url, pv, _, _ in result:
-        ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE))
+        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
     return ret

pym/euscan/scan.py

@@ -54,8 +54,6 @@ def parse_src_uri(uris):
 
         if '://' not in uri:
             continue
-        if 'mirror://' in uri:
-            uri = from_mirror(uri)
 
         if uris and uris[-1] == "->":
             operator = uris.pop()
@@ -102,7 +100,7 @@ def scan_upstream(query, on_progress=None):
     else:
         matches = Query(query).find(
             include_masked=True,
-            in_installed=False
+            in_installed=False,
         )
 
     if not matches:
@@ -165,9 +163,13 @@ def scan_upstream(query, on_progress=None):
         uris = pkg.environment('SRC_URI')
 
     cpv = pkg.cpv
-    urls = parse_src_uri(uris)
 
-    versions = handlers.scan(pkg, urls, on_progress)
+    uris = parse_src_uri(uris)
+    uris_expanded = [ from_mirror(uri) if 'mirror://' in uri else uri for uri in uris ]
+    pkg._uris = uris
+    pkg._uris_expanded = uris_expanded
+
+    versions = handlers.scan(pkg, uris, on_progress)
 
     cp, ver, rev = portage.pkgsplit(pkg.cpv)