Edit File: repos.py
#!/usr/bin/python -tt # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2004 Duke University import re import types import logging import misc import os import Errors from packageSack import MetaSack import urlgrabber.grabber from weakref import proxy as weakref class _wrap_ayum_getKeyForRepo: """ This is a wrapper for calling YumBase.getKeyForRepo() because otherwise we take a real reference through the bound method and that is d00m (this applies to YumBase and RepoStorage, hence why we have a separate class). A "better" fix might be to explicitly pass the YumBase instance to the callback ... API change! """ def __init__(self, ayum, ca=False): self.ayum = weakref(ayum) self.ca = ca def __call__(self, repo, callback=None): if self.ca: return self.ayum.getCAKeyForRepo(repo, callback) return self.ayum.getKeyForRepo(repo, callback) class RepoStorage: """This class contains multiple repositories and core configuration data about them.""" def __init__(self, ayum): self.repos = {} # list of repos by repoid pointing a repo object # of repo options/misc data self.callback = None # progress callback used for populateSack() for importing the xml files self.cache = 0 self.pkgSack = MetaSack() self.logger = logging.getLogger("yum.RepoStorage") self._setup = False self.ayum = weakref(ayum) # callbacks for handling gpg key imports for repomd.xml sig checks # need to be set from outside of the repos object to do anything # even quasi-useful # defaults to what is probably sane-ish self.gpg_import_func = _wrap_ayum_getKeyForRepo(ayum) self.gpgca_import_func = _wrap_ayum_getKeyForRepo(ayum, ca=True) self.confirm_func = None # This allow listEnabled() to be O(1) most of the time. self._cache_enabled_repos = [] self.quick_enable_disable = {} # This allows plugins to setup a repo. just before the first # listEnabled() call. self._list_enabled_hasrun = False def retrieveAllMD(self): """ Download metadata for all enabled repositories, based on mdpolicy. """ if not hasattr(urlgrabber.grabber, 'parallel_wait'): return repos = [] for repo in self.listEnabled(): if repo.cache: continue try: dl = repo._async and repo._commonLoadRepoXML(repo) except Errors.RepoError, e: if not repo.skip_if_unavailable: raise self.disableRepo(repo.id) dl = False if dl: mdtypes = repo._mdpolicy2mdtypes() downloading = repo._commonRetrieveDataMD_list(mdtypes) repos.append((repo, downloading, [False])) # with sizes first, then without sizes.. for no_size in (False, True): for repo, downloading, error in repos: def failfunc(obj, error=error): error[0] = True for (ndata, nmdtype) in downloading: if (ndata.size is None) == no_size: repo._retrieveMD(nmdtype, async=True, failfunc=failfunc) urlgrabber.grabber.parallel_wait() # done or revert for repo, downloading, error in repos: if error[0]: # some MD failed? repo._revertOldRepoXML() else: repo._commonRetrieveDataMD_done(downloading) def doSetup(self, thisrepo = None): if thisrepo is None: # Just in case the prelistenabledrepos plugin point hasn't run. self.listEnabled() self.ayum.plugins.run('prereposetup') if thisrepo is None: repos = self.listEnabled() else: repos = self.findRepos(thisrepo) if len(repos) < 1: self.logger.debug('No Repositories Available to Set Up') if hasattr(urlgrabber.grabber, 'pycurl'): # Must do basename checking, on cert. files... cert_basenames = {} for repo in self.listEnabled(): if repo.sslclientcert: bn = os.path.basename(repo.sslclientcert) other = cert_basenames.setdefault(bn, repo) if repo.sslclientcert != other.sslclientcert: msg = 'sslclientcert basename shared between %s and %s' raise Errors.ConfigError, msg % (repo, other) for repo in repos: repo.setup(self.ayum.conf.cache, self.ayum.mediagrabber, gpg_import_func = self.gpg_import_func, confirm_func=self.confirm_func, gpgca_import_func = self.gpgca_import_func) # if we come back from setup NOT enabled then mark as disabled # so nothing else touches us if not repo.enabled: self.disableRepo(repo.id) else: pkgdir = getattr(self.ayum.conf, 'downloaddir', None) if pkgdir: repo.pkgdir = pkgdir self._setup = True self.retrieveAllMD() self.ayum.plugins.run('postreposetup') def __str__(self): return str(self.repos.keys()) def __del__(self): try: self.close() except Errors.RepoError, e: self.logger.debug("Exception %s %s in %s ignored" % (repr(e), str(e), self.__del__)) def close(self): for repo in self.repos.values(): repo.close() def add(self, repoobj): if repoobj.id in self.repos: raise Errors.DuplicateRepoError, 'Repository %s is listed more than once in the configuration' % (repoobj.id) self.repos[repoobj.id] = repoobj if hasattr(repoobj, 'quick_enable_disable'): self.quick_enable_disable.update(repoobj.quick_enable_disable) repoobj.quick_enable_disable = self.quick_enable_disable else: self._cache_enabled_repos = None # At least pulp reuses RepoStorage but doesn't have a "real" YumBase() # so we can't guarantee new YumBase() attrs. exist. if not hasattr(self.ayum, '_override_sigchecks'): repoobj._override_sigchecks = False else: repoobj._override_sigchecks = self.ayum._override_sigchecks def delete(self, repoid): if repoid in self.repos: thisrepo = self.repos[repoid] thisrepo.close() del self.repos[repoid] def sort(self): repolist = self.repos.values() repolist.sort() return repolist def getRepo(self, repoid): try: return self.repos[repoid] except KeyError, e: raise Errors.RepoError, \ 'Error getting repository data for %s, repository not found' % (repoid) def findRepos(self, pattern, name_match=False, ignore_case=False): """ Find all repositories matching fnmatch `pattern` on the repo.id, can also do case insensitive searches and/or search on the name.""" if pattern in self.repos: # Minor opt. as we do this a lot... return [self.repos[pattern]] result = [] for item in pattern.split(','): item = item.strip() match = misc.compile_pattern(item.strip(), ignore_case) for name,repo in self.repos.items(): assert name == repo.id if match(name): result.append(repo) elif name_match and match(repo.name): result.append(repo) return result def disableRepo(self, repoid): """disable a repository from use fnmatch wildcards may be used to disable a group of repositories. returns repoid of disabled repos as list """ repos = [] if misc.re_glob(repoid) or repoid.find(',') != -1: for repo in self.findRepos(repoid): repos.append(repo.id) repo.disable() else: thisrepo = self.getRepo(repoid) repos.append(thisrepo.id) thisrepo.disable() return repos def enableRepo(self, repoid): """enable a repository for use fnmatch wildcards may be used to enable a group of repositories. returns repoid of enables repos as list """ repos = [] if misc.re_glob(repoid) or repoid.find(',') != -1: for repo in self.findRepos(repoid): repos.append(repo.id) repo.enable() else: thisrepo = self.getRepo(repoid) repos.append(thisrepo.id) thisrepo.enable() return repos def listEnabled(self): """return list of enabled repo objects""" if not self._list_enabled_hasrun: self.ayum.plugins.run('prelistenabledrepos') self._list_enabled_hasrun = True if (self._cache_enabled_repos is not None and not self.quick_enable_disable): return self._cache_enabled_repos returnlist = [] for repo in self.repos.values(): if repo.isEnabled(): returnlist.append(repo) returnlist.sort() if self._cache_enabled_repos is not None: self._cache_enabled_repos = returnlist self.quick_enable_disable.clear() return returnlist def listGroupsEnabled(self): """return a list of repo objects that have groups enabled""" returnlist = [] for repo in self.listEnabled(): if repo.enablegroups: returnlist.append(repo) return returnlist def setCache(self, cacheval): """sets cache value in all repos""" self.cache = cacheval for repo in self.repos.values(): repo.cache = cacheval def setCacheDir(self, cachedir): """sets the cachedir value in all repos""" self._cachedir = cachedir for repo in self.repos.values(): if cachedir != repo.basecachedir: repo.old_base_cache_dir = repo.basecachedir repo.basecachedir = cachedir def setProgressBar(self, obj, multi_obj=None): """sets the progress bar for downloading files from repos""" for repo in self.repos.values(): repo.setCallback(obj, multi_obj) def setFailureCallback(self, obj): """sets the failure callback for all repos""" for repo in self.repos.values(): repo.setFailureObj(obj) def setMirrorFailureCallback(self, obj): """sets the failure callback for all mirrors""" for repo in self.repos.values(): repo.setMirrorFailureObj(obj) def setInterruptCallback(self, callback): for repo in self.repos.values(): repo.setInterruptCallback(callback) def getPackageSack(self): return self.pkgSack def populateSack(self, which='enabled', mdtype='metadata', callback=None, cacheonly=0): """ This populates the package sack from the repositories, two optional arguments: - which='repoid, enabled, all' - mdtype='metadata, filelists, otherdata, all' """ if not self._setup: self.doSetup() if not callback: callback = self.callback myrepos = [] if which == 'enabled': myrepos = self.listEnabled() elif which == 'all': myrepos = self.repos.values() else: if type(which) == types.ListType: for repo in which: if isinstance(repo, Repository): myrepos.append(repo) else: repobj = self.getRepo(repo) myrepos.append(repobj) elif type(which) == types.StringType: repobj = self.getRepo(which) myrepos.append(repobj) if mdtype == 'all': data = ['metadata', 'filelists', 'otherdata'] else: data = [ mdtype ] if hasattr(urlgrabber.grabber, 'parallel_wait'): # download all metadata in parallel for repo in myrepos: if repo.cache: continue if repo._async: sack = repo.getPackageSack() sack._retrieve_async(repo, data) urlgrabber.grabber.parallel_wait() for repo in myrepos: sack = repo.getPackageSack() try: sack.populate(repo, mdtype, callback, cacheonly) except TypeError, e: if not e.args[0].startswith('Parsing'): raise if mdtype in ['all', 'metadata'] and repo.skip_if_unavailable: self.disableRepo(repo.id) else: raise Errors.RepoError(e.args[0]) except Errors.RepoError, e: if mdtype in ['all', 'metadata'] and repo.skip_if_unavailable: self.disableRepo(repo.id) else: raise else: self.pkgSack.addSack(repo.id, sack) class Repository: """this is an actual repository object""" def __init__(self, repoid): self.id = repoid self.quick_enable_disable = {} self.disable() self._xml2sqlite_local = False def __cmp__(self, other): """ Sort base class repos. by alphanumeric on their id, also see __cmp__ in YumRepository(). """ if self.id > other.id: return 1 elif self.id < other.id: return -1 else: return 0 def __str__(self): return self.id def __hash__(self): return hash(self.id) def __del__(self): try: self.close() except Errors.RepoError, e: self.logger.debug("Exception %s %s in %s ignored" % (repr(e), str(e), self.__del__)) def _ui_id(self): """ Show self.id, so we can use it and override it. """ return self.id ui_id = property(_ui_id) def close(self): pass def setAttribute(self, key, value): """sets a generic attribute of this repository""" setattr(self, key, value) def getAttribute(self, key): return getattr(self, key, None) def isEnabled(self): enabled = self.getAttribute('enabled') return enabled is not None and enabled def enable(self): self.setAttribute('enabled', 1) self.quick_enable_disable[self.id] = True def disable(self): self.setAttribute('enabled', 0) self.quick_enable_disable[self.id] = False def getExcludePkgList(self): excludeList = self.getAttribute('exclude') return excludeList or [] def getIncludePkgList(self): includeList = self.getAttribute('includepkgs') return includeList or [] # Abstract interface def ready(self): raise NotImplementedError() def getGroupLocation(self): raise NotImplementedError() def getPackageSack(self): raise NotImplementedError() def setup(self, cache): raise NotImplementedError() def setCallback(self, callback): raise NotImplementedError() def setFailureObj(self, obj): raise NotImplementedError() def setMirrorFailureObj(self, obj): raise NotImplementedError() def getPackage(self, package, checkfunc = None, text = None, cache = True): raise NotImplementedError() def getHeader(self, package, checkfunc = None, reget = 'simple', cache = True): raise NotImplementedError()