euscanwww: Added feed for stabilization candidates. Still a prototype though

Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino 2013-02-26 09:33:25 +01:00
parent 4b84dd54b5
commit c55d09a031
8 changed files with 323 additions and 17 deletions

View File

@ -1,4 +1,5 @@
import json
from urllib import urlencode
from django.contrib.syndication.views import Feed, FeedDoesNotExist
from django.shortcuts import get_object_or_404
@ -8,11 +9,95 @@ from django.db.models import Q
from euscan.version import gentoo_unstable
from djeuscan.models import Package, Herd, Maintainer, VersionLog
from djeuscan.models import Package, Herd, Maintainer, Version, VersionLog
from euscan_accounts.helpers import get_profile
class StabilizationCandidatesFeed(Feed):
feed_type = Atom1Feed
author_name = 'euscan'
item_author_name = author_name
title = 'Stabilization candidates'
link = "/"
description = "Stabilization candidates"
ttl = 3600
def item_title(self, version):
return version.cpv()
def item_description(self, version):
cpv = version.cpv()
maintainers = herds = ""
if version.package.maintainers.all():
maintainers = "Maintainers: {}\n".format(
", ".join(version.package.maintainers.all())
)
if version.package.herds.all():
herds = "Herds: {}\n".format(
", ".join(version.package.herds.all())
)
bugs_link = "https://bugs.gentoo.org/buglist.cgi?quicksearch={}"
comment = """
This bug was filed via euscan
How much have you used the package in question?
Have you had any problems with the package?
emerge --info:
Other info:
{} {}
""".format(herds, maintainers)
comment = "\n".join([line.lstrip() for line in comment.split("\n")])
description = """
Added to tree: {date}<br/>
{herds} <br />
{maintainers}<br />
Open bugs for {cat_pn}:
<a href="{cat_bugs}">Show</a><br />
Open bugs for {pn}:
<a href="{pn_bugs}">Show</a><br />
<a href="{submit_bug}">File bug </a>
""".format(
date=version.stabilization_candidate,
herds=herds,
maintainers=maintainers,
cat_pn=version.package.category,
cat_bugs=bugs_link.format(version.package.category),
pn=version.package,
pn_bugs=bugs_link.format(version.package),
submit_bug='https://bugs.gentoo.org/enter_bug.cgi?{}'.format(
urlencode({
"product": 'Gentoo Linux',
"short_desc": "Stable request for {}".format(cpv),
"comment": comment,
"keywords": 'STABLEREQ',
})
),
)
return description
def item_link(self, version):
kwargs = {'category': version.package.category,
'package': version.package.name}
return "%s#version-%s" % (
reverse('djeuscan.views.package', kwargs=kwargs), version.tag
)
def item_categories(self):
return ["stabilization_candidate"]
def items(self):
versions = Version.objects.exclude(stabilization_candidate=None)
return versions.order_by("stabilization_candidate")
class BaseFeed(Feed):
feed_type = Atom1Feed
author_name = 'euscan'

View File

@ -3,13 +3,25 @@ from django.core.management.base import BaseCommand
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.misc import stabilization_candidates
from optparse import make_option
logger = logging.getLogger(__name__)
class Command(BaseCommand):
_overlays = {}
help = 'Collect stabilization candidates'
option_list = BaseCommand.option_list + (
make_option(
'-d',
'--days-to-candidate',
action='store_true',
dest='all',
default=30,
help='Minimum of days to be in tree for becoming stable.'
),
)
help = 'Collects stabilization candidates'
def handle(self, *args, **options):
set_verbosity_level(logger, options.get("verbosity", 1))

View File

@ -0,0 +1,189 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'Version.stabilization_candidate'
db.add_column('djeuscan_version', 'stabilization_candidate',
self.gf('django.db.models.fields.DateField')(default=None, null=True, blank=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'Version.stabilization_candidate'
db.delete_column('djeuscan_version', 'stabilization_candidate')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'djeuscan.category': {
'Meta': {'object_name': 'Category'},
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'})
},
'djeuscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'})
},
'djeuscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'})
},
'djeuscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'djeuscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
},
'djeuscan.overlay': {
'Meta': {'object_name': 'Overlay'},
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'overlay_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'})
},
'djeuscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.problemreport': {
'Meta': {'object_name': 'ProblemReport'},
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'message': ('django.db.models.fields.TextField', [], {}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'})
},
'djeuscan.refreshpackagequery': {
'Meta': {'object_name': 'RefreshPackageQuery'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False'})
},
'djeuscan.version': {
'Meta': {'unique_together': "(['package', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'handler': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '128', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'stabilization_candidate': ('django.db.models.fields.DateField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.versionlog': {
'Meta': {'object_name': 'VersionLog'},
'action': ('django.db.models.fields.IntegerField', [], {}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['djeuscan']

View File

@ -152,6 +152,9 @@ class Version(models.Model):
ebuild_path = models.CharField(blank=True, max_length=256)
metadata_path = models.CharField(blank=True, max_length=256)
# If this is not null then the version is a stabilization candidate
stabilization_candidate = models.DateField(blank=True, null=True, default=None)
class Meta:
unique_together = ['package', 'revision', 'version', 'overlay']

View File

@ -2,7 +2,7 @@ import os.path
import re
from datetime import datetime, timedelta
from gentoolkit.package import Package
from dateutil.parser import parse
from dateutil.parser import parse
from djeuscan.processing import FakeLogger
from djeuscan.models import Version
@ -10,22 +10,28 @@ from djeuscan.models import Version
def get_version_date(version, date_limit):
"""
Returns the datetime when the version was added to Portage, if less than date_limit
Returns the datetime when the version was added to Portage,
if less than date_limit
"""
changelog_path = os.path.join(os.path.dirname(version.ebuild_path), "ChangeLog")
changelog_path = os.path.join(
os.path.dirname(version.ebuild_path),
"ChangeLog"
)
if not os.path.exists(changelog_path):
return
with open(changelog_path) as changelog:
for line in changelog:
match = re.match(r"^\*([^\(]+) \((\d\d \w\w\w \d\d\d\d)\)\s*$", line)
match = re.match(
r"^\*([^\(]+) \((\d\d \w\w\w \d\d\d\d)\)\s*$", line
)
if match:
version_date = parse(match.group(2)).date()
if version_date < date_limit:
return version_date
def stabilization_candidates(date_limit=None, logger=None):
def stabilization_candidates(days_to_candidate=30, logger=None):
"""
Collect stabilization candidates
"""
@ -33,25 +39,28 @@ def stabilization_candidates(date_limit=None, logger=None):
if logger is None:
logger = FakeLogger()
if date_limit is None:
date_limit = (datetime.utcnow() - timedelta(days=30)).date()
date_diff = (datetime.utcnow() - timedelta(days=days_to_candidate))
date_limit = date_diff.date()
logger.info("Starting collecting stabilization candidates - date_limit=%s", str(date_limit))
logger.info("Collecting stabilization candidates - date_limit=%s",
str(date_limit))
# Set all versions to not be stabilization_candidates
#Version.objects.update(stabilization_candidate=False)
# For every version check if it's unstable.
# If it is then check if can be a stabilization candidate
for version in Version.objects.filter(overlay='gentoo').exclude(version='9999').exclude(version='99999999'):
versions = Version.objects.filter(overlay='gentoo')\
.exclude(version='9999').exclude(version='99999999')
for version in versions:
pkg = Package(version.cpv())
keywords = pkg.environment("KEYWORDS").split()
if all([x.startswith("~") for x in keywords]):
version_date = get_version_date(version, date_limit)
if version_date:
logger.info('+ [s] %s @ %s', version, version_date)
# XXX: What should we save? A flag and version_date? Just the date?
#version.stabilization_candidate = True
#version.save()
# XXX: What should we save? A flag and the date?Just the date?
version.stabilization_candidate = version_date
version.save()
logger.info("Finished collecting stabilization candidates")

View File

@ -366,7 +366,11 @@ def collect_stabilization_candidates():
"""
This task collects the packages that are stabilization candidates
"""
pass
logger = collect_stabilization_candidates.get_logger()
misc.stabilization_candidates(
days_to_stabilization=settings.DAYS_TO_STABILIZATION,
logger=logger
)
admin_tasks = [
@ -382,4 +386,5 @@ admin_tasks = [
send_update_email,
send_weekly_email,
send_monthly_email,
collect_stabilization_candidates,
]

View File

@ -10,8 +10,7 @@ from euscan_accounts.views import favourite_package, unfavourite_package, \
favourite_overlay, unfavourite_overlay, favourite_world, unfavourite_world
from djeuscan.feeds import PackageFeed, CategoryFeed, HerdFeed, \
MaintainerFeed, GlobalFeed, WorldScanFeed
MaintainerFeed, GlobalFeed, WorldScanFeed, StabilizationCandidatesFeed
admin_required = user_passes_test(lambda u: u.is_superuser)
@ -83,6 +82,8 @@ urlpatterns = patterns('djeuscan.views',
url(r'^$', 'index', name="index"),
url(r'^feed/$', GlobalFeed(), name='global_feed'),
url(r'^feed/stabilization_candidates$', StabilizationCandidatesFeed(),
name='stabilization_candidates_feed'),
url(r'^about/$', 'about', name="about"),
url(r'^about/api$', 'api', name="api"),
url(r'^about/feeds$', 'feeds', name="feeds"),

View File

@ -245,6 +245,8 @@ CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler"
TASKS_UPSTREAM_GROUPS = 32
TASKS_EMAIL_GROUPS = 10
DAYS_TO_STABILIZATION = 30
# LDAP authentication
# TODO: Test data - change me!
AUTH_LDAP_SERVER_URI = "ldap://localhost"