#!/usr/bin/env python
import os, sys, re
import urlparse
from types import UnicodeType, NoneType
import difflib
import email.Utils
if sys.version_info[:2] < (2,3):
raise RuntimeError, "You need at least Python 2.3"
import optparse
import svn.repos
import svn.fs
import svn.core
import svn.delta
try:
from EaseXML import *
except ImportError:
print 'You need EaseXML (> 0.2.0) to use subveRSSed'
print 'Check it out at http://easexml.base-art.net'
sys.exit()
# check EaseXML version
def needUpgrade(version):
print """\
The detected EaseXML package is a bit old : "%s".
You need EaseXML > 0.2.0. If you can't find it on EaseXML's
website, check out the latest SVN trunk at:
http://svn.base-art.net/public/easexml/trunk/
""" % version
sys.exit()
try:
from EaseXML.main import __version_info__ as easeXmlVersion
except ImportError:
# __version_info__ appeared in EaseXML 0.2.1
from EaseXML.main import __version__
needUpgrade(__version__)
else:
if easeXmlVersion < (0,2,1):
needUpgrade(easeXmlVersion)
REPOS = '/usr/svn'
TITLE = 'Subversion Repository'
OUTPUT_PATH = os.getcwd()
REVISION = None
LINK = 'http://some-url.com/feed.html'
HAVEDIFF = False
REVRANGE = 0
REVSLICE = 1000
RSSNBITEMS = 20
ENCODING = 'latin-1'
__author__ = "Philippe Normand (philippe dot normand at finix dot eu dot org)"
__version__ = '1.2'
__license__ = 'GPL'
__doc__ = """\
SVN revision history in RSS format and HTML output.
Inspired by svnstatus.py wrote by Aaron Brady.
Author: %(author)s
Version: %(version)s
SubveRSSed will write 1 RSS feed and some HTML output files.
Usage: %%prog [options]""" % { 'author': __author__, 'version': __version__}
header = '''\
%(title)s
%(title)s
'''
footer = '''
'''
class Rss(XMLObject):
_name = 'rdf:RDF'
_nodesOrder = [ 'channel','items' ]
xmlns = StringAttribute(default="http://purl.org/rss/1.0/")
rdfns = StringAttribute(name="xmlns:rdf",
default="http://www.w3.org/1999/02/22-rdf-syntax-ns#")
dcns = StringAttribute(name="xmlns:dc",
default="http://purl.org/dc/elements/1.1/")
contentns = StringAttribute(name="xmlns:content",
default="http://purl.org/rss/1.0/modules/content/")
channel = ItemNode('Channel')
items = ListNode('Item')
def save(self, rssPath):
f = open(rssPath,'w')
f.write(self.toXml())
f.close()
def get_last_stored_rev(self):
rev = 1
if len(self.items) > 0:
firstItem = self.items[0]
match = re.match('Revision (\d+)', firstItem.title)
if match:
rev = int(match.groups()[0])
return rev
class Channel(XMLObject):
_name = 'channel'
_nodesOrder = [ 'title', 'link', 'description', 'itemsIds' ]
about = StringAttribute(name="rdf:about")
title = TextNode()
link = TextNode(optional=True)
description = TextNode('SVN Repository version history')
itemsIds = ItemNode('ItemIdsList')
class ItemIdsList(XMLObject):
_name = 'items'
items = ItemNode('Seq')
class Seq(XMLObject):
_name = "rdf:seq"
items = ListNode('ItemId',name="rdf:seq")
class ItemId(XMLObject):
_name = "rdf:li"
resource = StringAttribute(name="rdf:resource")
class Item(XMLObject):
_name = 'item'
_nodesOrder = [ 'author', 'pubDate', 'title', 'link', 'description' ]
about = StringAttribute(name="rdf:about",optional=True)
title = TextNode()
link = TextNode(optional=True)
pubDate = TextNode(name="dc:date")
author = TextNode(optional=True)
description = TextNode(optional=True)
def get_slice(sliceRev, rev):
result = sliceRev
while rev > result:
result += sliceRev
return result
class MyOptionFormatter(optparse.TitledHelpFormatter):
def format_option(self, option):
if option.default != 'NODEFAULT':
option.help += " (default: %s)" % repr(option.default)
return optparse.TitledHelpFormatter.format_option(self, option)
class SVNHelper:
def __init__(self, fs_ptr, pool):
self.fs_ptr = fs_ptr
self.pool = pool
self._cache = {}
def get_youngest(self):
youngest = svn.fs.youngest_rev(self.fs_ptr, self.pool)
return youngest
def get_info(self, rev, encoding):
date = svn.fs.revision_prop(self.fs_ptr, rev, svn.core.SVN_PROP_REVISION_DATE, self.pool)
author = svn.fs.revision_prop(self.fs_ptr, rev, svn.core.SVN_PROP_REVISION_AUTHOR, self.pool)
log = svn.fs.revision_prop(self.fs_ptr, rev, svn.core.SVN_PROP_REVISION_LOG, self.pool)
timestamp = svn.core.svn_parse_date(date, 0, self.pool)
formattedtime = email.Utils.formatdate(timestamp[1] / 1000000)
formattedtime = unicode(formattedtime, encoding, errors='ignore')
log = unicode(log, encoding, errors='ignore')
authorstring = ''
if author:
authorstring = author
else:
authorstring = 'nobody'
authorstring = unicode(authorstring, encoding, errors='ignore')
return (authorstring, formattedtime, log)
def cat(self, path, rev):
"""dump the contents of a file"""
contents = ''
fs_ptr = self.fs_ptr
taskpool = self.pool
root = svn.fs.revision_root(fs_ptr, rev, taskpool)
if not len(path):
print "You must supply a file path."
return contents
kind = svn.fs.check_path(root, path, taskpool)
if kind == svn.core.svn_node_none:
print "Path '%s' does not exist." % path
return contents
if kind == svn.core.svn_node_dir:
print "Path '%s' is not a file." % path
return contents
filelen = svn.fs.file_length(root, path, taskpool)
stream = svn.fs.file_contents(root, path, taskpool)
read = 0
while read < filelen:
contents += svn.core.svn_stream_read(stream, int(svn.core.SVN_STREAM_CHUNK_SIZE))
read += len(contents)
contents += svn.core.svn_stream_read(stream, int(filelen))
return contents
def diff(self, text1, text2):
result = difflib.unified_diff(text1.splitlines(1), text2.splitlines(1))
return ''.join(result)
def get_diff(self, rev, encoding):
changes = self.get_changed_files(rev)
final_diff = ''
for path, action, base_rev in changes:
text1 = self.cat(path, base_rev)
text2 = self.cat(path, rev)
final_diff += self.diff(text1, text2)
final_diff = unicode(final_diff, encoding, errors="replace")
return final_diff
def get_changed_files(self, rev):
cached = self._cache.get(rev)
if cached:
return cached
fs_ptr = self.fs_ptr
pool = self.pool
root = svn.fs.revision_root(fs_ptr, rev, pool)
editor = svn.repos.RevisionChangeCollector(fs_ptr, rev, pool)
e_ptr, e_baton = svn.delta.make_editor(editor, pool)
svn.repos.svn_repos_replay(root, e_ptr, e_baton, pool)
changes = []
for path, change in editor.changes.items():
if not change.path:
action = 'D'
elif change.added:
if not change.base_path or not change.base_rev:
action = 'A'
else:
action = 'U'
else:
action = 'U'
changes.append([path, action, change.base_rev])
changes.sort()
self._cache[rev] = changes
return changes
def get_changed(self, rev, encoding):
lines = []
for path, action, base_rev in self.get_changed_files(rev):
line = unicode('%s %s' % (action, path), encoding, errors='replace')
lines.append(line)
return lines
def writeHtmlOutput(slices, options):
for fname, items in slices.iteritems():
fname = os.path.join(options.OUTPUT_PATH,fname[1:])
print 'Writing "%s"' % fname
if os.path.exists(fname):
f = open(fname)
lines = f.readlines()[:-2]
f.close()
else:
lines = [header % {'title':options.TITLE}]
for i in items:
itemId = int(i.title.split()[1])
lines.append(' %s
\n' % (itemId,i.title.encode('utf8')))
lines.append(' %s (%s)
\n' % (i.pubDate.encode('utf8'),
i.author.encode('utf8')))
if i.description is not None:
lines.append(i.description.encode('utf8'))
f = open(fname,'w')
f.write(''.join(lines))
f.write(footer)
def main(pool, options):
repos_ptr = svn.repos.svn_repos_open(options.REPOS, pool)
fs_ptr = svn.repos.svn_repos_fs(repos_ptr)
fname = options.TITLE.replace(' ','_')
rssPath = os.path.join(options.OUTPUT_PATH,fname + '.xml')
try:
f = open(rssPath)
except IOError:
rss = Rss(channel=Channel(link=options.LINK, title=options.TITLE,
description='Latest commits on %s' % options.TITLE,
about=options.LINK))
rss.channel.itemsIds = ItemIdsList()
rss.channel.itemsIds.items = Seq()
else:
data = f.read()
f.close()
rss = Rss.fromXml(data)
encoding = options.ENCODING
helper = SVNHelper(fs_ptr, pool)
lastStoredRev = rss.get_last_stored_rev()
revSlice = options.REVSLICE
l = urlparse.urlparse(options.LINK)
path, ext = os.path.splitext(l[2])
base = os.path.dirname(path)
path = os.path.basename(path)
firstRev = lastStoredRev
lastRev = helper.get_youngest()
if options.REVRANGE:
r = options.REVRANGE.split('-')
if len(r) == 2:
firstRev, lastRev = r
else:
firstRev = r[0]
elif options.REVISION:
firstRev = lastRev = int(options.REVISION)
slices = {}
if firstRev != lastRev:
print 'Processing from revision %s to %s' % (firstRev, lastRev)
for i in range(int(firstRev), int(lastRev)+1):
author, date, log = helper.get_info(i,encoding)
changed = helper.get_changed(i,encoding)
item = Item(title=u'Revision %d' % i, pubDate=date,author=author)
newItemLink = "#%s" % i
sli = get_slice(revSlice,i)
sliceFname = "%s_%s%s" % (path, sli, ext)
if LINK:
p = os.path.join(base, sliceFname)
item.link = "%s://%s%s%s" % (l[0],l[1], p, newItemLink)
item.about = item.link
newItemLink = item.link
itemId = ItemId(resource=newItemLink)
if options.HAVEDIFF == True:
diff = helper.get_diff(i,encoding)
d = u'%s %s
' % (log, diff)
else:
d = u'%s
\n' % (log)
for line in changed:
line = line.split()
if len(line) > 1:
k = line[0]
v = u''.join(line[1:])
d += u'%s
' % (k,v)
item.description = d
if item not in rss.items:
try:
slices[sliceFname].append(item)
except:
slices[sliceFname] = [ item ]
print 'Adding revision %s' % i
rss.items.insert(0,item)
rss.channel.itemsIds.items.items.insert(0,itemId)
if len(rss.items) > options.RSSNBITEMS:
del rss.items[-1]
del rss.channel.itemsIds.items.items[-1]
writeHtmlOutput(slices,options)
rss.save(rssPath)
if __name__ == '__main__':
parser = optparse.OptionParser(usage=__doc__,
formatter=MyOptionFormatter())
parser.add_option('-e','--encoding', action='store',
dest='ENCODING', type='string', default=ENCODING,
help='Encoding to use.')
parser.add_option('-r', '--repository', action='store',
dest="REPOS", type='string', default=REPOS,
help="Repository path.")
parser.add_option('-n', action='store',
dest="RSSNBITEMS", type='int', default=RSSNBITEMS,
help="How many items to store in the RSS feed.")
parser.add_option('-s', '--revision-slice', action='store',
dest="REVSLICE", type='string', default=REVSLICE,
help="How many items to store in each HTML file.")
parser.add_option('-o', '--output', action='store', type='string',
dest="OUTPUT_PATH", default=OUTPUT_PATH,
help="Directory where output files (feed, html files) are stored.")
parser.add_option('-t', '--title', action='store',dest="TITLE",
type="string", help="HTML title.",default=TITLE)
parser.add_option('-l', '--link', action='store',dest="LINK", type="string",
help="URL of the HTML file.",default=LINK)
parser.add_option('-d', '--diff', action='store_true',dest="HAVEDIFF",
help="Append diff in RSS description.",default=HAVEDIFF)
parser.add_option('--revision-range', action='store', type='string',dest="REVRANGE",
help="Revisions to parse (a single number (revision to begin) or range (from-to revision)).",
default=REVRANGE)
parser.add_option('--revision', action='store', type='int',dest="REVISION",
help="Revision to parse (this options is not compatible with --revision-range).",
default=REVISION)
options, args = parser.parse_args()
svn.core.run_app(main, options=options)