euscan: misc fixes

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
This commit is contained in:
Corentin Chary 2012-08-15 11:04:29 +02:00
parent 88b15c18d2
commit bbfcc3d6d5
10 changed files with 30 additions and 17 deletions

1
TODO
View File

@ -22,6 +22,7 @@ euscan
Site Handlers Site Handlers
------------- -------------
- store un-expanded SRC_URI and use it in handlers
- berlios - berlios
- sourceforge: http://sourceforge.net/api/file/index/project-name/vboxgtk/mtime/desc/limit/20/rss http://sourceforge.net/api/release/index/project-id/264534/rss - sourceforge: http://sourceforge.net/api/file/index/project-name/vboxgtk/mtime/desc/limit/20/rss http://sourceforge.net/api/release/index/project-id/264534/rss
- MySQL: should use http://downloads.mysql.com/archives/ - MySQL: should use http://downloads.mysql.com/archives/

View File

@ -38,7 +38,6 @@ from euscan.out import progress_bar
isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty()
isatty_stderr = os.environ.get('TERM') != 'dumb' and sys.stderr.isatty() isatty_stderr = os.environ.get('TERM') != 'dumb' and sys.stderr.isatty()
def exit_helper(status): def exit_helper(status):
if CONFIG["format"]: if CONFIG["format"]:
print(output.get_formatted_output()) print(output.get_formatted_output())

View File

@ -53,6 +53,14 @@ def emerge_sync(logger):
"--config-root", settings.PORTAGE_CONFIGROOT] "--config-root", settings.PORTAGE_CONFIGROOT]
return _launch_command(cmd, logger) return _launch_command(cmd, logger)
def emerge_metadata(logger):
"""
Launches an emerge --metadata
"""
cmd = ["emerge", "--metadata", "--root", settings.PORTAGE_ROOT,
"--config-root", settings.PORTAGE_CONFIGROOT]
return _launch_command(cmd, logger)
def layman_sync(logger, cache=True): def layman_sync(logger, cache=True):
""" """
@ -106,7 +114,8 @@ def update_portage_trees(logger=None):
logger.info("Running layman --sync") logger.info("Running layman --sync")
layman_sync(logger, cache=True) layman_sync(logger, cache=True)
#logger.info("Running emerge --regen") #logger.info("Running emerge --regen")
#emerge_regen() #emerge_regen(logger)
emerge_metadata(logger)
logger.info("Running eix-update") logger.info("Running eix-update")
eix_update(logger) eix_update(logger)
logger.info("Done!") logger.info("Done!")

View File

@ -91,8 +91,10 @@ class ScanMetadata(object):
try: try:
metadata = None metadata = None
pkg, metadata = self.metadata_from_db(query, pkg) pkg, metadata = self.metadata_from_db(query, pkg)
if not metadata: if not metadata:
pkg, metadata = self.metadata_from_portage(query, pkg) pkg, metadata = self.metadata_from_portage(query, pkg)
if not metadata: if not metadata:
return return
except Exception as e: except Exception as e:
@ -148,6 +150,7 @@ class ScanMetadata(object):
self.style.ERROR("Bad maintainer: '%s' '%s'" % \ self.style.ERROR("Bad maintainer: '%s' '%s'" % \
(maintainer.name, maintainer.email)) (maintainer.name, maintainer.email))
) )
pkg.save() pkg.save()
def store_herd(self, name, email): def store_herd(self, name, email):

View File

@ -223,7 +223,7 @@ def update_upstream():
def scan_package(package): def scan_package(package):
scan_portage([package], purge_packages=True, purge_versions=True) scan_portage([package], purge_packages=True, purge_versions=True)
scan_metadata([package]) scan_metadata([package])
scan_upstream([package]) scan_upstream([package], purge_versions=True)
return True return True
@ -254,7 +254,7 @@ def consume_refresh_queue(locked=False):
pkg = query.package pkg = query.package
query.delete() query.delete()
scan_package_user.delay(pkg) scan_package_user.delay(pkg)
logger.info('Done (%s)' % pkg) logger.info('Selected: %s' % pkg)
except IndexError: except IndexError:
pass pass
finally: finally:

View File

@ -222,7 +222,6 @@ ACCOUNT_ACTIVATION_DAYS = 7
RECAPTCHA_PUBLIC_KEY = "" RECAPTCHA_PUBLIC_KEY = ""
RECAPTCHA_PRIVATE_KEY = "" RECAPTCHA_PRIVATE_KEY = ""
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
EMAIL_FROM = 'noreply@iksaif.net'
# djeuscan tasks # djeuscan tasks
PORTAGE_ROOT = "/" PORTAGE_ROOT = "/"

View File

@ -12,7 +12,7 @@ CONFIDENCE = 90
PRIORITY = 90 PRIORITY = 90
berlios_regex = r"berlios.de/([^/]+)/([^/]+)" berlios_regex = r"mirror://berlios/([^/]+)/([^/]+)"
def can_handle(pkg, url=None): def can_handle(pkg, url=None):

View File

@ -32,6 +32,6 @@ def scan_url(pkg, url, options):
result = url_scan(pkg, base_url, file_pattern) result = url_scan(pkg, base_url, file_pattern)
ret = [] ret = []
for found_url, pv, _, _ in result: for url, pv, _, _ in result:
ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE)) ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret return ret

View File

@ -12,7 +12,7 @@ PRIORITY = 90
def can_handle(pkg, url=None): def can_handle(pkg, url=None):
return url and "sourceforge.net/" in url return url and "mirror://sourceforge/" in url
def scan_url(pkg, url, options): def scan_url(pkg, url, options):
@ -20,7 +20,7 @@ def scan_url(pkg, url, options):
cp, ver, rev = portage.pkgsplit(pkg.cpv) cp, ver, rev = portage.pkgsplit(pkg.cpv)
project, filename = re.search("sourceforge.net/([^/]+)/(?:.*/)?([^/]+)", project, filename = re.search("mirror://sourceforge/([^/]+)/(?:.*/)?([^/]+)",
url).groups() url).groups()
base_url = "http://qa.debian.org/watch/sf.php/%s" % project base_url = "http://qa.debian.org/watch/sf.php/%s" % project
file_pattern = regex_from_template( file_pattern = regex_from_template(
@ -30,6 +30,6 @@ def scan_url(pkg, url, options):
result = url_scan(pkg, base_url, file_pattern) result = url_scan(pkg, base_url, file_pattern)
ret = [] ret = []
for found_url, pv, _, _ in result: for url, pv, _, _ in result:
ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE)) ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret return ret

View File

@ -54,8 +54,6 @@ def parse_src_uri(uris):
if '://' not in uri: if '://' not in uri:
continue continue
if 'mirror://' in uri:
uri = from_mirror(uri)
if uris and uris[-1] == "->": if uris and uris[-1] == "->":
operator = uris.pop() operator = uris.pop()
@ -102,7 +100,7 @@ def scan_upstream(query, on_progress=None):
else: else:
matches = Query(query).find( matches = Query(query).find(
include_masked=True, include_masked=True,
in_installed=False in_installed=False,
) )
if not matches: if not matches:
@ -165,9 +163,13 @@ def scan_upstream(query, on_progress=None):
uris = pkg.environment('SRC_URI') uris = pkg.environment('SRC_URI')
cpv = pkg.cpv cpv = pkg.cpv
urls = parse_src_uri(uris)
versions = handlers.scan(pkg, urls, on_progress) uris = parse_src_uri(uris)
uris_expanded = [ from_mirror(uri) if 'mirror://' in uri else uri for uri in uris ]
pkg._uris = uris
pkg._uris_expanded = uris_expanded
versions = handlers.scan(pkg, uris, on_progress)
cp, ver, rev = portage.pkgsplit(pkg.cpv) cp, ver, rev = portage.pkgsplit(pkg.cpv)