SF.net SVN: gar:[23433] csw/mgar/gar/v2/lib/python
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sun Apr 20 11:51:07 CEST 2014
Revision: 23433
http://sourceforge.net/p/gar/code/23433
Author: wahwah
Date: 2014-04-20 09:51:06 +0000 (Sun, 20 Apr 2014)
Log Message:
-----------
mGAR / buildfarm: Maintainer activity report
This is the first iteration of the script. It only looks at the catalog
state. To know which maintainers actually require retiring, the list
should be cross-matched with data from mantis.
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/stale_packages_report.py
Added Paths:
-----------
csw/mgar/gar/v2/lib/python/activity.py
csw/mgar/gar/v2/lib/python/colors.py
csw/mgar/gar/v2/lib/python/maintainer_activity_report.py
Property Changed:
----------------
csw/mgar/gar/v2/lib/python/stale_packages_report.py
Added: csw/mgar/gar/v2/lib/python/activity.py
===================================================================
--- csw/mgar/gar/v2/lib/python/activity.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/activity.py 2014-04-20 09:51:06 UTC (rev 23433)
@@ -0,0 +1,111 @@
+"""Functions for activity reporting."""
+
+import datetime
+import dateutil.parser
+import logging
+
+from collections import namedtuple
+
+from lib.python import colors
+from lib.python import opencsw
+
+Maintainer = namedtuple('Maintainer',
+ ['username', 'pkgs', 'last_activity', 'last_activity_pkg', 'active'])
+
+INACTIVE_MAINTAINER_CUTOFF = 2
+STALE_PACKAGE_CUTOFF = 4
+STALE_FROM_COLOR = '#FFDDBB'
+STALE_TO_COLOR = '#F995A0'
+
+def RevDeps(pkgs):
+ revdeps = {}
+ for entry in pkgs:
+ revdeps.setdefault(entry['pkgname'], set())
+ for dep in entry['deps']:
+ revdeps.setdefault(dep, set()).add(entry['pkgname'])
+ return revdeps
+
+
+def ByPkgname(pkgs):
+ pkgs_by_pkgname = {}
+ for entry in pkgs:
+ pkgs_by_pkgname[entry['pkgname']] = entry
+ return pkgs_by_pkgname
+
+
+def ComputeMaintainerActivity(maintainers):
+ now = datetime.datetime.now()
+ activity_cutoff = now - datetime.timedelta(days=INACTIVE_MAINTAINER_CUTOFF*365)
+ stale_pkg_cutoff = now - datetime.timedelta(days=STALE_PACKAGE_CUTOFF*365)
+
+ for username in maintainers:
+ if maintainers[username].last_activity < activity_cutoff:
+ maintainers[username] = maintainers[username]._replace(active=False)
+ pkgs = maintainers[username].pkgs
+ for catalogname in pkgs:
+ pkgs[catalogname]['old'] = pkgs[catalogname]['date'] < stale_pkg_cutoff
+ # All packages by inactive maintainers are stale by definition
+ if not maintainers[username].active:
+ pkgs[catalogname]['old'] = True
+ age = now - pkgs[catalogname]['date']
+ years = '%.1f' % (age.days / 365.0)
+ pkgs[catalogname]['age'] = age
+ pkgs[catalogname]['years'] = years
+ after_cutoff = stale_pkg_cutoff - pkgs[catalogname]['date']
+ frac = after_cutoff.days / float(365 * 4)
+ pkgs[catalogname]['color'] = colors.IntermediateColor(
+ STALE_FROM_COLOR, STALE_TO_COLOR, frac)
+ return maintainers
+
+
+def Maintainers(pkgs):
+ """Process a catalog and compile data structures with activity stats.
+
+ Args:
+ pkgs: a list of packages as returned by the catalog timing REST endpoint
+ Returns:
+ maintainers, bad_dates
+ """
+ bad_dates = []
+ maintainers = {}
+ for entry in pkgs:
+ entry['maintainer'] = entry['maintainer'].split('@')[0]
+ parsed_fn = opencsw.ParsePackageFileName(entry['basename'])
+ dates_to_try = []
+ if 'REV' in parsed_fn['revision_info']:
+ dates_to_try.append(parsed_fn['revision_info']['REV'])
+ else:
+ logging.warning('{catalogname} did not have a REV=. '
+ 'Falling back to mtime.'.format(**entry))
+ dates_to_try.append(entry['mtime'])
+
+ for date_str in dates_to_try:
+ try:
+ date = dateutil.parser.parse(date_str)
+ break
+ except ValueError as exc:
+ logging.warning(exc)
+ logging.warning(
+ "WTF is {date} in {catalogname}? "
+ "Go home {maintainer}, you're drunk.".format(
+ catalogname=entry['catalogname'],
+ maintainer=entry['maintainer'],
+ date=date_str))
+ bad_dates.append(date_str)
+ continue
+ entry['date'] = date
+ maintainer = maintainers.setdefault(entry['maintainer'],
+ Maintainer(username=entry['maintainer'], pkgs={},
+ last_activity=datetime.datetime(1970, 1, 1, 0, 0),
+ last_activity_pkg=None,
+ active=True))
+ if entry['catalogname'] not in maintainer.pkgs:
+ maintainer.pkgs[entry['catalogname']] = entry
+ if maintainer.last_activity < date:
+ maintainer = maintainer._replace(last_activity=date)
+ maintainer = maintainer._replace(last_activity_pkg=entry)
+ maintainers[maintainer.username] = maintainer
+
+ maintainers = ComputeMaintainerActivity(maintainers)
+
+ return maintainers, bad_dates
Added: csw/mgar/gar/v2/lib/python/colors.py
===================================================================
--- csw/mgar/gar/v2/lib/python/colors.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/colors.py 2014-04-20 09:51:06 UTC (rev 23433)
@@ -0,0 +1,28 @@
+"""Color processing."""
+
+
+def MakeColorTuple(hc):
+ R, G, B = hc[1:3], hc[3:5], hc[5:7]
+ R, G, B = int(R, 16), int(G, 16), int(B, 16)
+ return R, G, B
+
+
+def IntermediateColor(startcol, targetcol, frac):
+ """Return an intermediate color.
+
+ Fraction can be any rational number, but only the 0-1 range produces
+ gradients.
+ """
+ if frac < 0:
+ frac = 0
+ if frac >= 1.0:
+ frac = 1.0
+ sc = MakeColorTuple(startcol)
+ tc = MakeColorTuple(targetcol)
+ dR = tc[0] - sc[0]
+ dG = tc[1] - sc[1]
+ dB = tc[2] - sc[2]
+ R = sc[0] + dR * frac
+ G = sc[1] + dG * frac
+ B = sc[2] + dB * frac
+ return "#%02x%02x%02x" % (R, G, B)
Added: csw/mgar/gar/v2/lib/python/maintainer_activity_report.py
===================================================================
--- csw/mgar/gar/v2/lib/python/maintainer_activity_report.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/maintainer_activity_report.py 2014-04-20 09:51:06 UTC (rev 23433)
@@ -0,0 +1,162 @@
+#!/opt/csw/bin/python2.7
+# coding: utf-8
+
+"""Generate a HTML report of activity of OpenCSW maintainers.
+
+It combines input from a few catalog releases to make sure we get a better
+image of activity of a maintainer.
+"""
+
+import logging
+import requests
+import argparse
+import datetime
+import jinja2
+import cPickle
+
+import concurrent.futures
+
+from lib.python import activity
+from lib.python import colors
+
+REPORT_TMPL = u"""<!DOCTYPE html>
+<html>
+<head>
+ <title>Maintainer activity report</title>
+ <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
+ <style TYPE="text/css">
+ body, p, li {
+ font-size: 14px;
+ font-family: sans-serif;
+ }
+ .active-False .maintainer {
+ color: brown;
+ }
+ .active-False .activity-tag {
+ color: brown;
+ font-weight: bold;
+ }
+ .active-True .activity-tag {
+ color: green;
+ font-weight: bold;
+ }
+ .warning {
+ color: red;
+ font-weight: bold;
+ }
+ .can-be-retired-True {
+ background-color: #FED;
+ }
+ </style>
+</head>
+<body>
+ <h1>Maintainer activity report</h1>
+ <p>
+ </p>
+ <table>
+ <tr>
+ <th>username</th>
+ <th>last activity</th>
+ <th>years</th>
+ <th>active?</th>
+ <th>last pkg</th>
+ <th># pkgs</th>
+ </tr>
+ {% for username in maintainers|sort %}
+ <tr class="active-{{ maintainers[username].active }} can-be-retired-{{ analysis_by_username[username].can_be_retired }}">
+ <td>
+ <a id="{{ username }}" class="maintainer"
+ href="http://www.opencsw.org/maintainers/{{ username }}/">{{ username }}</a>
+ </td>
+ <td>
+ {{ maintainers[username].last_activity.strftime('%Y-%m-%d') }}
+ </td>
+ <td>
+ {{ maintainers[username].last_activity_pkg.years }}
+ </td>
+ <td class="activity-tag">
+ {% if maintainers[username].active %}
+ <span class="activity-tag">active</span>
+ {% else %}
+ <span class="activity-tag">inactive</span>
+ {% endif %}
+ </td>
+ <td>
+ <a href="http://buildfarm.opencsw.org/pkgdb/srv4/{{ maintainers[username].last_activity_pkg.md5_sum }}/">
+ {{ maintainers[username].last_activity_pkg.catalogname }}</a>
+ </td>
+ <td>
+ {% if username in maintainers_in_unstable %}
+ {{ maintainers_in_unstable[username].pkgs|length }}
+ {% endif %}
+ </td>
+ </tr>
+ {% endfor %}
+ </table>
+</body>
+</html>
+"""
+
+def ConcurrentFetchResults(catrels):
+ def Fetch(catrel):
+ url = ('http://buildfarm.opencsw.org/pkgdb/rest/catalogs/'
+ '{}/i386/SunOS5.10/timing/'.format(catrel))
+ logging.debug('GetPkgs(%r)', url)
+ return requests.get(url).json()
+ results_by_catrel = {}
+ with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+ future_to_catrel = dict((executor.submit(Fetch, catrel), catrel)
+ for catrel in catrels)
+ for future in concurrent.futures.as_completed(future_to_catrel):
+ catrel = future_to_catrel[future]
+ if future.exception() is not None:
+ logging.warning('Fetching %r failed', url)
+ else:
+ results_by_catrel[catrel] = future.result()
+ return results_by_catrel
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('output', help='Output file (HTML)')
+ parser.add_argument('--save-as', dest="save_as",
+ help="Save data to file; useful when you're modifying the program and "
+ "you want to re-run quickly without fetching all the data again.")
+ parser.add_argument('--load-from', dest="load_from",
+ help='Load data from file')
+ args = parser.parse_args()
+ catrels = ['kiel', 'bratislava', 'unstable']
+ if args.load_from:
+ with open(args.load_from, 'r') as fd:
+ results_by_catrel = cPickle.load(fd)
+ else:
+ results_by_catrel = ConcurrentFetchResults(catrels)
+
+ if args.save_as:
+ with open(args.save_as, 'w') as fd:
+ cPickle.dump(results_by_catrel, fd)
+
+ pkgs = [item for sublist in results_by_catrel.values() for item in sublist]
+ maintainers, bad_dates = activity.Maintainers(pkgs)
+ maintainers_in_unstable, _ = activity.Maintainers(results_by_catrel['unstable'])
+
+ analysis_by_username = {}
+ for username, maintainer in maintainers.iteritems():
+ d = {'can_be_retired': False}
+ if not maintainer.active:
+ if username not in maintainers_in_unstable:
+ d['can_be_retired'] = True
+ analysis_by_username[username] = d
+
+ with open(args.output, 'w') as outfd:
+ template = jinja2.Template(REPORT_TMPL)
+ rendered = template.render(
+ maintainers=maintainers,
+ maintainers_in_unstable=maintainers_in_unstable,
+ analysis_by_username=analysis_by_username)
+ outfd.write(rendered.encode('utf-8'))
+
+
+if __name__ == '__main__':
+ logging.basicConfig(level=logging.DEBUG)
+ main()
Property changes on: csw/mgar/gar/v2/lib/python/maintainer_activity_report.py
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
Modified: csw/mgar/gar/v2/lib/python/stale_packages_report.py
===================================================================
--- csw/mgar/gar/v2/lib/python/stale_packages_report.py 2014-04-19 15:19:54 UTC (rev 23432)
+++ csw/mgar/gar/v2/lib/python/stale_packages_report.py 2014-04-20 09:51:06 UTC (rev 23433)
@@ -1,4 +1,4 @@
-#!/opt/csw/bin/python
+#!/opt/csw/bin/python2.7
# coding: utf-8
"""Retrieve catalog state and generate a HTML report of stale packages."""
@@ -6,22 +6,16 @@
import logging
import requests
import argparse
-import dateutil.parser
import datetime
import jinja2
-from collections import namedtuple
+from lib.python import activity
-from lib.python import opencsw
-
-INACTIVE_MAINTAINER_CUTOFF = 2
-STALE_PACKAGE_CUTOFF = 4
REMOVE_SUGGESTION_CUTOFF = 4
-STALE_FROM_COLOR = '#FFDDBB'
-STALE_TO_COLOR = '#F995A0'
-REPORT_TMPL = u"""<html>
+REPORT_TMPL = u"""<!DOCTYPE html>
+<html>
<head>
- <title>Packages to rebuild</title>
+ <title>Stale packages report</title>
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
<style TYPE="text/css">
body, p, li {
@@ -57,6 +51,12 @@
href="https://docs.google.com/document/d/1a5aTPXk5qxnuOng6o2FaQtgr3uqqtEIbCN23WHQgaOc/edit#">OpenCSW Wintercamp in Zürich</a>.
</p>
<ul>
+ <li>
+ <a href="#could-be-dropped">Package candidates for deleting from the catalog</a>
+ </li>
+ </ul>
+ <h2>Stale packages by maintainer</h2>
+ <ul>
{% for username in maintainers|sort %}
<li class="{% if maintainers[username].active %}active{% else %}inactive{% endif %}">
<a id="{{ username }}" class="maintainer" href="http://www.opencsw.org/maintainers/{{ username }}/">{{ username }}</a>
@@ -78,7 +78,7 @@
>
{{ catalogname }}</a>
<span style="background: {{ maintainers[username].pkgs[catalogname].color }};">
- ({{ maintainers[username].pkgs[catalogname].age }}
+ ({{ maintainers[username].pkgs[catalogname].years }}
years{% if revdeps[maintainers[username].pkgs[catalogname].pkgname] %},
{% if revdeps[maintainers[username].pkgs[catalogname].pkgname]|length > 5 %}
{{ revdeps[maintainers[username].pkgs[catalogname].pkgname]|length }} revdeps
@@ -100,7 +100,8 @@
</li>
{% endfor %}
</ul>
- <h1>Packages that could be dropped now</h1>
+ <a href="#" id="could-be-dropped"></a>
+ <h2>Packages that could be dropped now</h2>
<p>
Here's a list of packages that: (1) have inactive maintainers, (2) are older
than {{ remove_suggestion_cutoff }} years, (3) have no reverse dependencies.
@@ -118,131 +119,20 @@
"""There was a bad date tag."""
-Maintainer = namedtuple('Maintainer',
- ['username', 'pkgs', 'last_activity', 'last_activity_pkg', 'active'])
-
-
-
-def MakeColorTuple(hc):
- R, G, B = hc[1:3], hc[3:5], hc[5:7]
- R, G, B = int(R, 16), int(G, 16), int(B, 16)
- return R, G, B
-
-
-def IntermediateColor(startcol, targetcol, frac):
- """Return an intermediate color.
-
- Fraction can be any rational number, but only the 0-1 range produces
- gradients.
- """
- if frac < 0:
- frac = 0
- if frac >= 1.0:
- frac = 1.0
- sc = MakeColorTuple(startcol)
- tc = MakeColorTuple(targetcol)
- dR = tc[0] - sc[0]
- dG = tc[1] - sc[1]
- dB = tc[2] - sc[2]
- R = sc[0] + dR * frac
- G = sc[1] + dG * frac
- B = sc[2] + dB * frac
- return "#%02x%02x%02x" % (R, G, B)
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('output', help='Output file')
- args = parser.parse_args()
- url = ('http://buildfarm.opencsw.org/pkgdb/rest/catalogs/'
- 'unstable/i386/SunOS5.10/timing/')
- data = requests.get(url).json()
-
- bad_dates = []
- revdeps = {}
- maintainers = {}
+def StalePkgsReport(output_file_name, revdeps, maintainers, pkgs_by_pkgname):
packages_to_drop = []
- for entry in data:
- entry['maintainer'] = entry['maintainer'].split('@')[0]
- parsed_fn = opencsw.ParsePackageFileName(entry['basename'])
- dates_to_try = []
- if 'REV' in parsed_fn['revision_info']:
- dates_to_try.append(parsed_fn['revision_info']['REV'])
- else:
- logging.warning('{catalogname} did not have a REV=. '
- 'Falling back to mtime.'.format(**entry))
- dates_to_try.append(entry['mtime'])
-
- for date_str in dates_to_try:
- try:
- date = dateutil.parser.parse(date_str)
- break
- except ValueError as exc:
- logging.warning(exc)
- logging.warning(
- "WTF is {date} in {catalogname}? "
- "Go home {maintainer}, you're drunk.".format(
- catalogname=entry['catalogname'],
- maintainer=entry['maintainer'],
- date=date_str))
- bad_dates.append(date_str)
- continue
- entry['date'] = date
- maintainer = maintainers.setdefault(entry['maintainer'],
- Maintainer(username=entry['maintainer'], pkgs={},
- last_activity=datetime.datetime(1970, 1, 1, 0, 0),
- last_activity_pkg=None,
- active=True))
- if entry['catalogname'] not in maintainer.pkgs:
- maintainer.pkgs[entry['catalogname']] = entry
- if maintainer.last_activity < date:
- maintainer = maintainer._replace(last_activity=date)
- maintainer = maintainer._replace(last_activity_pkg=entry)
- maintainers[maintainer.username] = maintainer
- revdeps.setdefault(entry['pkgname'], set())
- for dep in entry['deps']:
- revdeps.setdefault(dep, set()).add(entry['pkgname'])
- del entry
- if bad_dates:
- logging.warning('Bad dates encountered. mtime used as fallback.')
- now = datetime.datetime.now()
- activity_cutoff = now - datetime.timedelta(days=INACTIVE_MAINTAINER_CUTOFF*365)
- stale_pkg_cutoff = now - datetime.timedelta(days=STALE_PACKAGE_CUTOFF*365)
-
- # Make an index by pkgname
- pkgs_by_pkgname = {}
- for maintainer in maintainers.itervalues():
- for catalogname in maintainer.pkgs:
- entry = maintainer.pkgs[catalogname]
- pkgs_by_pkgname[entry['pkgname']] = entry
-
for username in maintainers:
- if maintainers[username].last_activity < activity_cutoff:
- maintainers[username] = maintainers[username]._replace(active=False)
pkgs = maintainers[username].pkgs
for catalogname in pkgs:
- pkgs[catalogname]['old'] = pkgs[catalogname]['date'] < stale_pkg_cutoff
- # All packages by inactive maintainers are stale by definition
- if not maintainers[username].active:
- pkgs[catalogname]['old'] = True
- age = now - pkgs[catalogname]['date']
- years = '%.1f' % (age.days / 365.0)
- pkgs[catalogname]['age'] = years
- after_cutoff = stale_pkg_cutoff - pkgs[catalogname]['date']
- frac = after_cutoff.days / float(365 * 4)
- pkgs[catalogname]['color'] = IntermediateColor(
- STALE_FROM_COLOR, STALE_TO_COLOR, frac)
-
# Package dropping logic
entry = pkgs[catalogname]
maintainer = maintainers[username]
- if (age > datetime.timedelta(days=REMOVE_SUGGESTION_CUTOFF*365) and
+ if (entry['age'] > datetime.timedelta(days=REMOVE_SUGGESTION_CUTOFF*365) and
not revdeps[entry['pkgname']] and
not maintainer.active):
packages_to_drop.append(entry)
# Find packages to rebuild
- #
for username in maintainers:
pkgs = maintainers[username].pkgs
for catalogname in pkgs:
@@ -252,14 +142,27 @@
revdep = pkgs_by_pkgname[revdep_pkgname]
if not revdep['old'] and entry['old']:
entry['rebuild'] = True
- with open(args.output, 'wb') as fd:
+ with open(output_file_name, 'wb') as fd:
template = jinja2.Template(REPORT_TMPL)
fd.write(template.render(maintainers=maintainers, revdeps=revdeps,
- inactive_maint_cutoff=INACTIVE_MAINTAINER_CUTOFF,
- stale_pkg_cutoff=STALE_PACKAGE_CUTOFF,
+ inactive_maint_cutoff=activity.INACTIVE_MAINTAINER_CUTOFF,
+ stale_pkg_cutoff=activity.STALE_PACKAGE_CUTOFF,
packages_to_drop=packages_to_drop,
remove_suggestion_cutoff=REMOVE_SUGGESTION_CUTOFF).encode('utf-8'))
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('output', help='Output file')
+ args = parser.parse_args()
+ url = ('http://buildfarm.opencsw.org/pkgdb/rest/catalogs/'
+ 'unstable/i386/SunOS5.10/timing/')
+ pkgs = requests.get(url).json()
+ maintainers, bad_dates = activity.Maintainers(pkgs)
+ revdeps = activity.RevDeps(pkgs)
+ pkgs_by_pkgname = activity.ByPkgname(pkgs)
+ StalePkgsReport(args.output, revdeps, maintainers, pkgs_by_pkgname)
+
+
if __name__ == '__main__':
main()
Property changes on: csw/mgar/gar/v2/lib/python/stale_packages_report.py
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list