Edit File: update_md.py
#!/usr/bin/python -t # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2005 Duke University # # Seth Vidal <skvidal@linux.duke.edu> # Luke Macken <lmacken@redhat.com> """ Update metadata (updateinfo.xml) parsing. """ import sys from yum.i18n import utf8_text_wrap, to_utf8, to_unicode, _ from yum.yumRepo import YumRepository from yum.packages import FakeRepository from yum.misc import to_xml, decompress, repo_gen_decompress from yum.misc import cElementTree_iterparse as iterparse import Errors import logginglevels import rpmUtils.miscutils from rpmUtils.arch import ArchStorage def safe_iterparse(filename, logger=None): """ Works like iterparse, but hides XML errors (prints a warning). """ try: for event, elem in iterparse(filename): yield event, elem except SyntaxError: # Bad XML if logger: logger.critical(_("Updateinfo file is not valid XML: %s"), filename) else: print >> sys.stderr, "Updateinfo file is not valid XML:", filename class UpdateNoticeException(Exception): """ An exception thrown for bad UpdateNotice data. """ pass class UpdateNotice(object): """ A single update notice (for instance, a security fix). """ def __init__(self, elem=None, repoid=None, vlogger=None): self._md = { 'from' : '', 'type' : '', 'title' : '', 'release' : '', 'status' : '', 'version' : '', 'pushcount' : '', 'update_id' : '', 'issued' : '', 'updated' : '', 'description' : '', 'rights' : '', 'severity' : '', 'summary' : '', 'solution' : '', 'references' : [], 'pkglist' : [], 'reboot_suggested' : False } if elem: self._parse(elem) self._repoid = repoid self._vlogger = vlogger def __getitem__(self, item): """ Allows scriptable metadata access (ie: un['update_id']). """ if type(item) is int: return sorted(self._md)[item] ret = self._md.get(item) if ret == '': ret = None return ret def __contains__(self, item): """ Allows quick tests for foo in blah. """ return item in self._md def __setitem__(self, item, val): self._md[item] = val def __eq__(self, other): # Tests to see if it's "the same data", which means that the # packages can be different (see add_notice). def _rid(un): if hasattr(un, '_repoid') and un._repoid is not None: return un._repoid else: return '<unknown>' def _log_failure(data): """Log the mismatched data similarly to conflict markers in git.""" if self._vlogger is None: return msg = _('Duplicate of %s differs in some fields:\n') msg %= other._md['update_id'] msg += '<<<<<<< %s:%s\n' % (_rid(other), data) msg += '%r\n=======\n%r\n' % (other._md[data], self._md[data]) msg += '>>>>>>> %s:%s' % (_rid(self), data) # --verbose mode enables this self._vlogger.log(logginglevels.DEBUG_3, msg) if not other or not hasattr(other, '_md'): return False for data in ('type', 'update_id', 'status', 'rights', 'severity', 'release', 'issued', 'updated', 'version', 'pushcount', 'from', 'title', 'summary', 'description', 'solution'): if data == 'status': # FIXME: See below... continue if self._md[data] != other._md[data]: _log_failure(data) return False # FIXME: Massive hack, Fedora is really broken and gives status=stable # and status=testing for updateinfo notices, just depending on which # repo. they come from. data = 'status' if self._md[data] != other._md[data]: if self._md[data] not in ('stable', 'testing'): _log_failure(data) return False if other._md[data] not in ('stable', 'testing'): _log_failure(data) return False # They are both really "stable" ... self._md[data] = 'stable' other._md[data] = 'stable' return True def __ne__(self, other): return not (self == other) def text(self, skip_data=('files', 'summary', 'rights', 'solution')): head = """ =============================================================================== %(title)s =============================================================================== Update ID : %(update_id)s Release : %(release)s Type : %(type)s Status : %(status)s Issued : %(issued)s """ % self._md if self._md['updated'] and self._md['updated'] != self._md['issued']: head += " Updated : %s" % self._md['updated'] # Add our bugzilla references bzs = filter(lambda r: r['type'] == 'bugzilla', self._md['references']) if len(bzs) and 'bugs' not in skip_data: buglist = " Bugs :" for bz in bzs: buglist += " %s%s\n\t :" % (bz['id'], 'title' in bz and ' - %s' % bz['title'] or '') head += buglist[: - 1].rstrip() + '\n' # Add our CVE references cves = filter(lambda r: r['type'] == 'cve', self._md['references']) if len(cves) and 'cves' not in skip_data: cvelist = " CVEs :" for cve in cves: cvelist += " %s\n\t :" % cve['id'] head += cvelist[: - 1].rstrip() + '\n' if self._md['summary'] and 'summary' not in skip_data: data = utf8_text_wrap(self._md['summary'], width=64, subsequent_indent=' ' * 12 + ': ') head += " Summary : %s\n" % '\n'.join(data) if self._md['description'] and 'description' not in skip_data: desc = utf8_text_wrap(self._md['description'], width=64, subsequent_indent=' ' * 12 + ': ') head += "Description : %s\n" % '\n'.join(desc) if self._md['solution'] and 'solution' not in skip_data: data = utf8_text_wrap(self._md['solution'], width=64, subsequent_indent=' ' * 12 + ': ') head += " Solution : %s\n" % '\n'.join(data) if self._md['rights'] and 'rights' not in skip_data: data = utf8_text_wrap(self._md['rights'], width=64, subsequent_indent=' ' * 12 + ': ') head += " Rights : %s\n" % '\n'.join(data) if self._md['severity'] and 'severity' not in skip_data: data = utf8_text_wrap(self._md['severity'], width=64, subsequent_indent=' ' * 12 + ': ') head += " Severity : %s\n" % '\n'.join(data) if 'files' in skip_data: return head[:-1] # chop the last '\n' # Get a list of arches we care about: #XXX ARCH CHANGE - what happens here if we set the arch - we need to # pass this in, perhaps arches = set(rpmUtils.arch.getArchList()) filelist = " Files :" for pkg in self._md['pkglist']: for file in pkg['packages']: if file['arch'] not in arches: continue filelist += " %s\n\t :" % file['filename'] head += filelist[: - 1].rstrip() return head def __str__(self): return to_utf8(self.text()) def __unicode__(self): return to_unicode(self.text()) def get_metadata(self): """ Return the metadata dict. """ return self._md def _parse(self, elem): """ Parse an update element:: <!ELEMENT update (id, synopsis?, issued, updated, references, description, rights?, severity?, summary?, solution?, pkglist)> <!ATTLIST update type (errata|security) "errata"> <!ATTLIST update status (final|testing) "final"> <!ATTLIST update version CDATA #REQUIRED> <!ATTLIST update from CDATA #REQUIRED> """ if elem.tag == 'update': for attrib in ('from', 'type', 'status', 'version'): self._md[attrib] = elem.attrib.get(attrib) for child in elem: if child.tag == 'id': if not child.text: raise UpdateNoticeException("No id element found") self._md['update_id'] = child.text elif child.tag == 'pushcount': self._md['pushcount'] = child.text elif child.tag == 'issued': self._md['issued'] = child.attrib.get('date') elif child.tag == 'updated': self._md['updated'] = child.attrib.get('date') elif child.tag == 'references': self._parse_references(child) elif child.tag == 'description': self._md['description'] = child.text elif child.tag == 'rights': self._md['rights'] = child.text elif child.tag == 'severity': self._md[child.tag] = child.text elif child.tag == 'summary': self._md['summary'] = child.text elif child.tag == 'solution': self._md['solution'] = child.text elif child.tag == 'pkglist': self._parse_pkglist(child) elif child.tag == 'title': self._md['title'] = child.text elif child.tag == 'release': self._md['release'] = child.text else: raise UpdateNoticeException('No update element found') def _parse_references(self, elem): """ Parse the update references:: <!ELEMENT references (reference*)> <!ELEMENT reference> <!ATTLIST reference href CDATA #REQUIRED> <!ATTLIST reference type (self|other|cve|bugzilla) "self"> <!ATTLIST reference id CDATA #IMPLIED> <!ATTLIST reference title CDATA #IMPLIED> """ for reference in elem: if reference.tag == 'reference': data = {} for refattrib in ('id', 'href', 'type', 'title'): data[refattrib] = reference.attrib.get(refattrib) self._md['references'].append(data) else: raise UpdateNoticeException('No reference element found') def _parse_pkglist(self, elem): """ Parse the package list:: <!ELEMENT pkglist (collection+)> <!ELEMENT collection (name?, package+)> <!ATTLIST collection short CDATA #IMPLIED> <!ATTLIST collection name CDATA #IMPLIED> <!ELEMENT name (#PCDATA)> """ for collection in elem: data = { 'packages' : [] } if 'short' in collection.attrib: data['short'] = collection.attrib.get('short') for item in collection: if item.tag == 'name': data['name'] = item.text elif item.tag == 'package': data['packages'].append(self._parse_package(item)) self._md['pkglist'].append(data) def _parse_package(self, elem): """ Parse an individual package:: <!ELEMENT package (filename, sum, reboot_suggested)> <!ATTLIST package name CDATA #REQUIRED> <!ATTLIST package version CDATA #REQUIRED> <!ATTLIST package release CDATA #REQUIRED> <!ATTLIST package arch CDATA #REQUIRED> <!ATTLIST package epoch CDATA #REQUIRED> <!ATTLIST package src CDATA #REQUIRED> <!ELEMENT reboot_suggested (#PCDATA)> <!ELEMENT filename (#PCDATA)> <!ELEMENT sum (#PCDATA)> <!ATTLIST sum type (md5|sha1) "sha1"> """ package = {} for pkgfield in ('arch', 'epoch', 'name', 'version', 'release', 'src'): package[pkgfield] = elem.attrib.get(pkgfield) # Bad epoch and arch data is the most common (missed) screwups. # Deal with bad epoch data. if not package['epoch'] or package['epoch'][0] not in '0123456789': package['epoch'] = None for child in elem: if child.tag == 'filename': package['filename'] = child.text elif child.tag == 'sum': package['sum'] = (child.attrib.get('type'), child.text) elif child.tag == 'reboot_suggested': self._md['reboot_suggested'] = True return package def xml(self): """Generate the xml for this update notice object""" msg = """ <update from="%s" status="%s" type="%s" version="%s"> <id>%s</id> <title>%s</title> <release>%s</release> <issued date="%s"/> <description>%s</description>\n""" % (to_xml(self._md['from']), to_xml(self._md['status']), to_xml(self._md['type']), to_xml(self._md['version']), to_xml(self._md['update_id']), to_xml(self._md['title']), to_xml(self._md['release']), to_xml(self._md['issued'], attrib=True), to_xml(self._md['description'])) if self._md['updated']: # include the updated date in the generated xml msg += """ <updated date="%s"/>\n""" % (to_xml(self._md['updated'], attrib=True)) if self._md['summary']: msg += """ <summary>%s</summary>\n""" % (to_xml(self._md['summary'])) if self._md['solution']: msg += """ <solution>%s</solution>\n""" % (to_xml(self._md['solution'])) if self._md['rights']: msg += """ <rights>%s</rights>\n""" % (to_xml(self._md['rights'])) if self._md['severity']: msg += """ <severity>%s</severity>\n""" % (to_xml(self._md['severity'])) if self._md['references']: msg += """ <references>\n""" for ref in self._md['references']: if ref['title']: msg += """ <reference href="%s" id="%s" title="%s" type="%s"/>\n""" % ( to_xml(ref['href'], attrib=True), to_xml(ref['id'], attrib=True), to_xml(ref['title'], attrib=True), to_xml(ref['type'], attrib=True)) else: msg += """ <reference href="%s" id="%s" type="%s"/>\n""" % ( to_xml(ref['href'], attrib=True), to_xml(ref['id'], attrib=True), to_xml(ref['type'], attrib=True)) msg += """ </references>\n""" if self._md['pkglist']: msg += """ <pkglist>\n""" for coll in self._md['pkglist']: msg += """ <collection short="%s">\n <name>%s</name>\n""" % ( to_xml(coll['short'], attrib=True), to_xml(coll['name'])) for pkg in coll['packages']: msg += """ <package arch="%s" name="%s" release="%s" src="%s" version="%s" epoch="%s"> <filename>%s</filename> </package>\n""" % (to_xml(pkg['arch'], attrib=True), to_xml(pkg['name'], attrib=True), to_xml(pkg['release'], attrib=True), to_xml(pkg['src'], attrib=True), to_xml(pkg['version'], attrib=True), to_xml(pkg['epoch'] or '0', attrib=True), to_xml(pkg['filename'])) msg += """ </collection>\n""" msg += """ </pkglist>\n""" msg += """</update>\n""" return msg def _rpm_tup_vercmp(tup1, tup2): """ Compare two "std." tuples, (n, a, e, v, r). """ return rpmUtils.miscutils.compareEVR((tup1[2], tup1[3], tup1[4]), (tup2[2], tup2[3], tup2[4])) class UpdateMetadata(object): """ The root update metadata object. """ def __init__(self, repos=[], logger=None, vlogger=None): self._notices = {} self._cache = {} # a pkg nvr => notice cache for quick lookups self._no_cache = {} # a pkg name only => notice list self._repos = [] # list of repo ids that we've parsed self._logger = logger self._vlogger = vlogger for repo in repos: try: # attempt to grab the updateinfo.xml.gz from the repodata self.add(repo) except Errors.RepoMDError: continue # No metadata found for this repo self.arch_storage = ArchStorage() self.archlist = self.arch_storage.archlist def get_notices(self, name=None): """ Return all notices. """ if name is None: return self._notices.values() return name in self._no_cache and self._no_cache[name] or [] notices = property(get_notices) def get_notice(self, nvr): """ Retrieve an update notice for a given (name, version, release) string or tuple. """ if type(nvr) in (type([]), type(())): nvr = '-'.join(nvr) return self._cache.get(nvr) or None # The problem with the above "get_notice" is that not everyone updates # daily. So if you are at pkg-1, pkg-2 has a security notice, and pkg-3 # has a BZ fix notice. All you can see is the BZ notice for the new "pkg-3" # with the above. # So now instead you lookup based on the _installed_ pkg.pkgtup, and get # two notices, in order: [(pkgtup-3, notice), (pkgtup-2, notice)] # the reason for the sorting order is that the first match will give you # the minimum pkg you need to move to. def get_applicable_notices(self, pkgtup): """ Retrieve any update notices which are newer than a given std. pkgtup (name, arch, epoch, version, release) tuple. Returns: list of (pkgtup, notice) that are newer than the given pkgtup, in the order of newest pkgtups first. """ oldpkgtup = pkgtup name = oldpkgtup[0] arch = oldpkgtup[1] ret = [] other_arch_list = [] notices = set() for notice in self.get_notices(name): for upkg in notice['pkglist']: for pkg in upkg['packages']: other_arch = False if pkg['name'] != name or pkg['arch'] != arch: if (notice not in notices and pkg['name'] == name and pkg['arch'] in self.archlist): other_arch = True else: continue pkgtup = (pkg['name'], pkg['arch'], pkg['epoch'] or '0', pkg['version'], pkg['release']) if _rpm_tup_vercmp(pkgtup, oldpkgtup) <= 0: continue if other_arch: other_arch_list.append((pkgtup, notice)) else: ret.append((pkgtup, notice)) notices.add(notice) for pkgtup, notice in other_arch_list: if notice not in notices: ret.append((pkgtup, notice)) ret.sort(cmp=_rpm_tup_vercmp, key=lambda x: x[0], reverse=True) return ret def add_notice(self, un): """ Add an UpdateNotice object. This should be fully populated with data, esp. update_id and pkglist/packages. """ if not un or not un["update_id"]: return False # This is "special", the main thing we want to deal with here is # having one errata that has multiple packages in it rpmA and rpmB, but # the packages are in repos. repoA and repoB. So instead of doing a # single errata pointing to both rpmA and rpmB and put the same thing # in both repodata (which is legal, and works fine) people want to have # just the packages from repoA in the repodata for repoA and vice versa. if un['update_id'] in self._notices: oun = self._notices[un['update_id']] if oun != un: return False # Ok, main parts of errata are the same, so now merge references: seen = set() for ref in oun['references']: seen.add(ref['id']) for ref in un['references']: if ref['id'] in seen: continue seen.add(ref['id']) oun['references'].append(ref) # ...and pkglist (this assumes that a pkglist name XYZ is the same): seen = set() for pkg in oun['pkglist']: seen.add(pkg['name']) for pkg in un['pkglist']: if pkg['name'] in seen: continue seen.add(pkg['name']) oun['pkglist'].append(pkg) un = oun self._notices[un['update_id']] = un for pkg in un['pkglist']: for filedata in pkg['packages']: self._cache['%s-%s-%s' % (filedata['name'], filedata['version'], filedata['release'])] = un no = self._no_cache.setdefault(filedata['name'], set()) no.add(un) return True def add(self, obj, mdtype='updateinfo'): """ Parse a metadata from a given YumRepository, file, or filename. """ def _rid(repoid, fmt=_(' (from %s)')): if not repoid: return '' return fmt % repoid if not obj: raise UpdateNoticeException repoid = None if type(obj) in (type(''), type(u'')): unfile = decompress(obj) infile = open(unfile, 'rt') elif isinstance(obj, YumRepository): if obj.id not in self._repos: repoid = obj.id self._repos.append(obj.id) md = obj.retrieveMD(mdtype) if not md: raise UpdateNoticeException() unfile = repo_gen_decompress(md, 'updateinfo.xml') infile = open(unfile, 'rt') elif isinstance(obj, FakeRepository): raise Errors.RepoMDError, "No updateinfo for local pkg" else: # obj is a file object infile = obj have_dup = False for event, elem in safe_iterparse(infile, logger=self._logger): if elem.tag == 'update': try: un = UpdateNotice(elem, repoid, self._vlogger) except UpdateNoticeException, e: msg = _("An update notice%s is broken, skipping.") % _rid(repoid) if self._vlogger: self._vlogger.log(logginglevels.DEBUG_1, "%s", msg) else: print >> sys.stderr, msg continue if not self.add_notice(un): msg = _("Update notice %s%s is broken, or a bad duplicate, skipping.") % (un['update_id'], _rid(repoid)) if not have_dup: msg += _('\nYou should report this problem to the owner of the %srepository.') % _rid(repoid, "%s ") msg += _('\nTo help pinpoint the issue, please attach the output of "yum updateinfo --verbose" to the report.') have_dup = True if self._vlogger: self._vlogger.warn("%s", msg) else: print >> sys.stderr, msg def __unicode__(self): ret = u'' for notice in self.notices: ret += unicode(notice) return ret def __str__(self): return to_utf8(self.__unicode__()) def xml(self, fileobj=None): msg = """<?xml version="1.0"?>\n<updates>""" if fileobj: fileobj.write(msg) for notice in self._notices.values(): if fileobj: fileobj.write(notice.xml()) else: msg += notice.xml() end = """</updates>\n""" if fileobj: fileobj.write(end) else: msg += end if fileobj: return return msg def main(): """ update_md test function. """ import yum.misc yum.misc.setup_locale() def usage(): print >> sys.stderr, "Usage: %s <update metadata> ..." % sys.argv[0] sys.exit(1) if len(sys.argv) < 2: usage() try: print sys.argv[1] um = UpdateMetadata() for srcfile in sys.argv[1:]: um.add(srcfile) print unicode(um) except IOError: print >> sys.stderr, "%s: No such file:\'%s\'" % (sys.argv[0], sys.argv[1:]) usage() if __name__ == '__main__': main()