Edit File: sqlitesack.py
#!/usr/bin/python -tt # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2005 Duke University # # Implementation of the YumPackageSack class that uses an sqlite backend # import os import os.path import fnmatch import yumRepo from packages import PackageObject, RpmBase, YumAvailablePackage, parsePackages import Errors import misc from sqlutils import executeSQL, sql_esc, sql_esc_glob import rpmUtils.miscutils import sqlutils import constants import operator from yum.misc import seq_max_split from yum.i18n import to_utf8, to_unicode import sys import re import warnings def catchSqliteException(func): """This decorator converts sqlite exceptions into RepoError""" def newFunc(*args, **kwargs): try: return func(*args, **kwargs) except sqlutils.sqlite.Error, e: # 2.4.x requires this, but 2.6.x complains about even hasattr() # of e.message ... *sigh* if sys.hexversion < 0x02050000: if hasattr(e,'message'): raise Errors.RepoError, str(e.message) else: raise Errors.RepoError, str(e) raise Errors.RepoError, str(e) newFunc.__name__ = func.__name__ newFunc.__doc__ = func.__doc__ newFunc.__dict__.update(func.__dict__) return newFunc def _share_data(value): return misc.share_data(value) # FIXME: parsePackages() def _parse_pkg_n(match, regexp_match, n): if match == n: return True if not regexp_match: return False if (match and n and match[0] not in ('?', '*', '[') and match[0] != n[0]): return False if regexp_match(n): return True return False def _parse_pkg(match, regexp_match, data, e,v,r,a): n = data['n'] assert e, 'Nothing in epoch' # Worthless speed hacks? if match == n: return True if (match and n and match[0] not in ('?', '*', '[') and match[0] != n[0] and match[0] != e[0]): return False if 'nameArch' not in data: data['nameArch'] = '%s.%s' % (n, a) data['nameVerRelArch'] = '%s-%s-%s.%s' % (n, v, r, a) data['nameVer'] = '%s-%s' % (n, v) data['nameVerRel'] = '%s-%s-%s' % (n, v, r) data['envra'] = '%s:%s-%s-%s.%s' % (e, n, v, r, a) data['nevra'] = '%s-%s:%s-%s.%s' % (n, e, v, r, a) data = set([n, data['nameArch'], data['nameVerRelArch'], data['nameVer'], data['nameVerRel'], data['envra'], data['nevra']]) if match in data: return True if not regexp_match: return False for item in data: if regexp_match(item): return True return False def _excluder_match(excluder, match, regexp_match, data, e,v,r,a): if False: pass elif excluder in ('eq', 'match'): if _parse_pkg(match, regexp_match, data, e,v,r,a): return True elif excluder in ('name.eq', 'name.match'): if _parse_pkg_n(match, regexp_match, data['n']): return True elif excluder in ('arch.eq', 'arch.match'): if _parse_pkg_n(match, regexp_match, a): return True elif excluder == 'nevr.eq': if 'nevr' not in data: data['nevr'] = '%s-%s:%s-%s' % (data['n'], e, v, r) if match == data['nevr']: return True elif excluder in ('nevra.eq', 'nevra.match'): if 'nevra' not in data: data['nevra'] = '%s-%s:%s-%s.%s' % (data['n'], e, v, r, a) if _parse_pkg_n(match, regexp_match, data['nevra']): return True elif excluder == 'name.in': if data['n'] in match: return True elif excluder == 'nevr.in': if 'nevr' not in data: data['nevr'] = '%s-%s:%s-%s' % (data['n'], e, v, r) if data['nevr'] in match: return True elif excluder == 'nevra.in': if 'nevra' not in data: data['nevra'] = '%s-%s:%s-%s.%s' % (data['n'], e, v, r, a) if data['nevra'] in match: return True elif excluder == 'pkgtup.eq': if match == data['pkgtup']: return True elif excluder == 'pkgtup.in': if data['pkgtup'] in match: return True elif excluder == 'marked': if data['marked']: return True elif excluder == 'washed': if not data['marked']: return True elif excluder == '*': return True else: assert False, 'Bad excluder: ' + excluder return None return False def _deduplicate(cur, field): """Eliminate duplicate rows from cursor based on field. Assuming the result set can be divided into one or more equivalent groups of rows based on the given field, this wrapper will yield rows from only one of the groups, avoiding duplicates. """ first_val = None for ob in cur: if first_val is None: first_val = ob[field] elif ob[field] != first_val: continue yield ob class YumAvailablePackageSqlite(YumAvailablePackage, PackageObject, RpmBase): def __init__(self, repo, db_obj): self.prco = { 'obsoletes': (), 'conflicts': (), 'requires': (), 'provides': () } self.sack = repo.sack self.repoid = repo.id self.repo = repo self.state = None self._loadedfiles = False self._files = None self._read_db_obj(db_obj) # for stupid metadata created without epochs listed in the version tag # die die if self.epoch is None: self.epoch = '0' self.id = self.pkgId self.ver = self.version self.rel = self.release self.pkgtup = (self.name, self.arch, self.epoch, self.version, self.release) self._changelog = None self._hash = None files = property(fget=lambda self: self._loadFiles()) def _read_db_obj(self, db_obj, item=None): """read the db obj. If asked for a specific item, return it. otherwise populate out into the object what exists""" if item: try: return db_obj[item] except (IndexError, KeyError): return None for item in ['name', 'arch', 'epoch', 'version', 'release', 'pkgKey']: try: setattr(self, item, _share_data(db_obj[item])) except (IndexError, KeyError): pass try: self.pkgId = db_obj['pkgId'] checksum_type = _share_data(db_obj['checksum_type']) check_sum = (checksum_type, db_obj['pkgId'], True) self._checksums = [ check_sum ] except (IndexError, KeyError): pass @catchSqliteException def _sql_MD(self, MD, sql, *args): """ Exec SQL against an MD of the repo, return a cursor. """ cache = getattr(self.sack, MD + 'db')[self.repo] cur = cache.cursor() executeSQL(cur, sql, *args) return cur def __getattr__(self, varname): db2simplemap = { 'packagesize' : 'size_package', 'archivesize' : 'size_archive', 'installedsize' : 'size_installed', 'buildtime' : 'time_build', 'hdrstart' : 'rpm_header_start', 'hdrend' : 'rpm_header_end', 'basepath' : 'location_base', 'relativepath': 'location_href', 'filetime' : 'time_file', 'packager' : 'rpm_packager', 'group' : 'rpm_group', 'buildhost' : 'rpm_buildhost', 'sourcerpm' : 'rpm_sourcerpm', 'vendor' : 'rpm_vendor', 'license' : 'rpm_license', 'checksum_value' : 'pkgId', } # If these existed, then we wouldn't get here ... and nothing in the DB # starts and ends with __'s. So these are missing. if varname.startswith('__') and varname.endswith('__'): raise AttributeError, varname dbname = db2simplemap.get(varname, varname) try: r = self._sql_MD('primary', "SELECT %s FROM packages WHERE pkgId = ?" % dbname, (self.pkgId,)).fetchone() except Errors.RepoError, e: if str(e).startswith('no such column'): #FIXME - after API break make this an AttributeError Raise raise KeyError, str(e) raise value = r[0] if varname == 'epoch' and value is None: value = '0' if varname in ('summary', 'description') and value is None: # Maybe others here? ... location_base is a bad NONO though. value = '' # Description for picasa, probably among others *sigh* if varname in {'vendor' : 1, 'packager' : 1, 'buildhost' : 1, 'license' : 1, 'group' : 1, 'summary' : 1, 'description' : 1, 'sourcerpm' : 1, 'url' : 1}: value = _share_data(value) setattr(self, varname, value) return value # Note that we use pkgId instead of pkgKey to filter the files and # changelog entries since we can't guarantee that pkgKeys in primarydb and # filelistsdb are in sync (since self.pkgKey is obtained from primarydb). # # Also, because of that, we must make sure not to return duplicate entries # in case we have some duplicate packages (i.e. same checksums), so we use # _deduplicate(). def _loadFiles(self): if self._loadedfiles: return self._files result = {} #FIXME - this should be try, excepting self.sack.populate(self.repo, mdtype='filelists') cur = self._sql_MD('filelists', "SELECT pkgKey, dirname, filetypes, filenames " \ "FROM filelist JOIN packages USING(pkgKey) " \ "WHERE packages.pkgId = ?", (self.pkgId,)) for ob in _deduplicate(cur, 'pkgKey'): dirname = ob['dirname'] if dirname == '.': dirname = '' elif dirname and dirname[-1] != '/': dirname += '/' filetypes = decodefiletypelist(ob['filetypes']) filenames = decodefilenamelist(ob['filenames']) while(filetypes): filename = dirname + filenames.pop() filetype = _share_data(filetypes.pop()) result.setdefault(filetype,[]).append(filename) self._loadedfiles = True self._files = result return self._files def _loadChangelog(self): result = [] if not self._changelog: if self.repo not in self.sack.otherdb: try: self.sack.populate(self.repo, mdtype='otherdata') except Errors.RepoError: self._changelog = result return cur = self._sql_MD('other', "SELECT pkgKey, date, author, changelog " \ "FROM changelog JOIN packages USING(pkgKey) " \ "WHERE pkgId = ? ORDER BY date DESC", (self.pkgId,)) # Check count(pkgId) here, the same way we do in searchFiles()? # Failure mode is much less of a problem. for ob in _deduplicate(cur, 'pkgKey'): # Note: Atm. rpm only does days, where (60 * 60 * 24) == 86400 # and we have the hack in _dump_changelog() to keep the # order the same, so this is a quick way to get rid of # any extra "seconds". # We still leak the seconds if there are 100 updates in # a day ... but don't do that. It also breaks if rpm ever # gets fixed (but that is unlikely). c_date = 100 * (ob['date'] / 100) c_author = to_utf8(ob['author']) c_log = to_utf8(ob['changelog']) result.append((c_date, _share_data(c_author), c_log)) self._changelog = result return def returnIdSum(self): return (self.checksum_type, self.pkgId) def returnChangelog(self): self._loadChangelog() return self._changelog def returnFileEntries(self, ftype='file', primary_only=False): """return list of files based on type, you can pass primary_only=True to limit to those files in the primary repodata""" if primary_only and not self._loadedfiles: sql = "SELECT name as fname FROM files WHERE pkgKey = ? and type = ?" cur = self._sql_MD('primary', sql, (self.pkgKey, ftype)) return map(lambda x: x['fname'], cur) self._loadFiles() return RpmBase.returnFileEntries(self,ftype,primary_only) def returnFileTypes(self, primary_only=False): """return list of types of files in the package, you can pass primary_only=True to limit to those files in the primary repodata""" if primary_only and not self._loadedfiles: sql = "SELECT DISTINCT type as ftype FROM files WHERE pkgKey = ?" cur = self._sql_MD('primary', sql, (self.pkgKey,)) return map(lambda x: x['ftype'], cur) self._loadFiles() return RpmBase.returnFileTypes(self) def simpleFiles(self, ftype='file'): warnings.warn('simpleFiles() will go away in a future version of Yum.' 'Use returnFileEntries(primary_only=True)\n', Errors.YumDeprecationWarning, stacklevel=2) sql = "SELECT name as fname FROM files WHERE pkgKey = ? and type = ?" cur = self._sql_MD('primary', sql, (self.pkgKey, ftype)) return map(lambda x: x['fname'], cur) def returnPrco(self, prcotype, printable=False): prcotype = _share_data(prcotype) if prcotype == 'strong_requires': # pkg not installed so we don't know require flags yet # returning all requires should work in most cases prcotype = 'requires' if isinstance(self.prco[prcotype], tuple): sql = "SELECT name, version, release, epoch, flags " \ "FROM %s WHERE pkgKey = ?" % prcotype cur = self._sql_MD('primary', sql, (self.pkgKey,)) self.prco[prcotype] = [ ] for ob in cur: if not ob['name']: continue prco_set = (_share_data(ob['name']), _share_data(ob['flags']), (_share_data(ob['epoch']), _share_data(ob['version']), _share_data(ob['release']))) self.prco[prcotype].append(_share_data(prco_set)) return RpmBase.returnPrco(self, prcotype, printable) def _requires_with_pre(self): """returns requires with pre-require bit""" sql = "SELECT name, version, release, epoch, flags,pre " \ "FROM requires WHERE pkgKey = ?" cur = self._sql_MD('primary', sql, (self.pkgKey,)) requires = [] for ob in cur: pre = "0" if ob['pre'].lower() in ['true', 1]: pre = "1" prco_set = (_share_data(ob['name']), _share_data(ob['flags']), (_share_data(ob['epoch']), _share_data(ob['version']), _share_data(ob['release'])), pre) requires.append(prco_set) return requires class YumSqlitePackageSack(yumRepo.YumPackageSack): """ Implementation of a PackageSack that uses sqlite cache instead of fully expanded metadata objects to provide information """ def __init__(self, packageClass): # Just init as usual and create a dict to hold the databases yumRepo.YumPackageSack.__init__(self, packageClass) self.primarydb = {} self.filelistsdb = {} self.otherdb = {} self.excludes = {} # of [repo] => {} of pkgId's => 1 self._excludes = set() # of (repo, pkgKey) self._exclude_whitelist = set() # of (repo, pkgKey) self._all_excludes = {} self._search_cache = { 'provides' : { }, 'requires' : { }, } self._key2pkg = {} self._pkgname2pkgkeys = {} self._pkgtup2pkgs = {} self._pkgnames_loaded = set() self._pkgmatch_fails = set() self._provmatch_fails = set() self._arch_allowed = None self._pkgExcluder = [] self._pkgExcludeIds = {} self._pkgobjlist_dirty = False @catchSqliteException def _sql_MD(self, MD, repo, sql, *args): """ Exec SQL against an MD of the repo, return a cursor. """ cache = getattr(self, MD + 'db')[repo] cur = cache.cursor() executeSQL(cur, sql, *args) return cur def _sql_MD_pkg_num(self, MD, repo): """ Give a count of pkgIds in the given repo DB """ sql = "SELECT count(pkgId) FROM packages" return self._sql_MD('primary', repo, sql).fetchone()[0] def _clean_pkgobjlist(self): """ If the pkgobjlist is dirty (possible pkgs on it which are excluded) then clean it, and return the clean list. """ assert hasattr(self, 'pkgobjlist') if self._pkgobjlist_dirty: pol = filter(lambda x: not self._pkgExcluded(x), self.pkgobjlist) self.pkgobjlist = pol self._pkgobjlist_dirty = False return self.pkgobjlist def __len__(self): # First check if everything is excluded all_excluded = True for (repo, cache) in self.primarydb.items(): if repo not in self._all_excludes: all_excluded = False break if all_excluded: return 0 if hasattr(self, 'pkgobjlist'): return len(self._clean_pkgobjlist()) exclude_num = 0 for repo in self.excludes: exclude_num += len(self.excludes[repo]) pkg_num = 0 for repo in self.primarydb: pkg_num += self._sql_MD_pkg_num('primary', repo) return pkg_num - exclude_num def dropCachedData(self): if hasattr(self, '_memoize_requires'): del self._memoize_requires if hasattr(self, '_memoize_provides'): del self._memoize_provides if hasattr(self, 'pkgobjlist'): del self.pkgobjlist self._pkgobjlist_dirty = False self._key2pkg = {} self._pkgname2pkgkeys = {} self._pkgnames_loaded = set() self._pkgmatch_fails = set() self._provmatch_fails = set() self._pkgtup2pkgs = {} self._search_cache = { 'provides' : { }, 'requires' : { }, } misc.unshare_data() @catchSqliteException def close(self): self.dropCachedData() for dataobj in self.primarydb.values() + \ self.filelistsdb.values() + \ self.otherdb.values(): dataobj.close() self.primarydb = {} self.filelistsdb = {} self.otherdb = {} self.excludes = {} self._excludes = set() self._exclude_whitelist = set() self._all_excludes = {} self._pkgExcluder = [] self._pkgExcludeIds = {} self._pkgobjlist_dirty = False yumRepo.YumPackageSack.close(self) def buildIndexes(self): # We don't need to play with returnPackages() caching as it handles # additions to excludes after the cache is built. pass def _checkIndexes(self, failure='error'): return def _delPackageRK(self, repo, pkgKey): ''' Exclude a package so that _pkgExcluded*() knows it's gone. Note that this doesn't update self.exclude. ''' self._excludes.add((repo, pkgKey)) # Don't keep references around, just wastes memory. if repo in self._key2pkg: po = self._key2pkg[repo].pop(pkgKey, None) if po is not None: # Will also be in the pkgtup2pkgs cache... pos = self._pkgtup2pkgs[po.pkgtup] pos = filter(lambda x: id(x) == id(po), pos) self._pkgtup2pkgs[po.pkgtup] = pos # Remove a package # Because we don't want to remove a package from the database we just # add it to the exclude list def delPackage(self, obj): if obj.repo not in self.excludes: self.excludes[obj.repo] = {} self.excludes[obj.repo][obj.pkgId] = 1 if (obj.repo, obj.pkgKey) in self._exclude_whitelist: self._exclude_whitelist.discard((obj.repo, obj.pkgKey)) self._delPackageRK(obj.repo, obj.pkgKey) self._pkgobjlist_dirty = True def _delAllPackages(self, repo): """ Exclude all packages from the repo. """ self._all_excludes[repo] = True if repo in self.excludes: del self.excludes[repo] if repo in self._key2pkg: del self._key2pkg[repo] if repo in self._pkgname2pkgkeys: del self._pkgname2pkgkeys[repo] def _excluded(self, repo, pkgId): if repo in self._all_excludes: return True if repo in self.excludes and pkgId in self.excludes[repo]: return True return False def _pkgKeyExcluded(self, repo, pkgKey): if self._all_excludes and repo in self._all_excludes: return True return self._excludes and (repo, pkgKey) in self._excludes def _pkgExcludedRKNEVRA(self, repo,pkgKey, n,e,v,r,a): ''' Main function to use for "can we use this package" question. . Tests repo against allowed repos. . Tests pkgKey against allowed packages. . Tests arch against allowed arches. . Tests addPackageExcluder() calls. ''' if self._exclude_whitelist and (repo,pkgKey) in self._exclude_whitelist: return False if self._pkgKeyExcluded(repo, pkgKey): return True if self._arch_allowed is not None and a not in self._arch_allowed: self._delPackageRK(repo, pkgKey) return True if not self._pkgExcluder: return False data = {'n' : n.lower(), 'pkgtup' : (n, a, e, v, r), 'marked' : False} e = e.lower() v = v.lower() r = r.lower() a = a.lower() for repoid, excluder, match, regexp_match in self._pkgExcluder: if repoid is not None and repoid != repo.id: continue exSPLIT = excluder.split('.', 1) if len(exSPLIT) != 2: assert False, 'Bad excluder: ' + excluder continue exT, exM = exSPLIT if False: pass elif exT == 'exclude': if _excluder_match(exM, match, regexp_match, data, e,v,r,a): self._delPackageRK(repo, pkgKey) return True elif exT == 'include': if _excluder_match(exM, match, regexp_match, data, e,v,r,a): break elif exT == 'mark': if data['marked']: pass # Speed opt. don't do matches we don't need to do. elif _excluder_match(exM, match, regexp_match, data, e,v,r,a): data['marked'] = True elif exT == 'wash': if not data['marked']: pass # Speed opt. don't do matches we don't need to do. elif _excluder_match(exM, match, regexp_match, data, e,v,r,a): data['marked'] = False else: assert False, 'Bad excluder: ' + excluder self._exclude_whitelist.add((repo, pkgKey)) return False def _pkgExcludedRKT(self, repo,pkgKey, pkgtup): ''' Helper function to call _pkgExcludedRKNEVRA. Takes a repo, pkgKey and a package tuple''' (n,a,e,v,r) = pkgtup return self._pkgExcludedRKNEVRA(repo, pkgKey, n,e,v,r,a) def _pkgExcludedRKD(self, repo,pkgKey, data): ''' Helper function to call _pkgExcludedRKNEVRA. Takes a repo, pkgKey and a dict of package data''' (n,a,e,v,r) = (data['name'], data['arch'], data['epoch'], data['version'], data['release']) return self._pkgExcludedRKNEVRA(repo, pkgKey, n,e,v,r,a) def _pkgExcluded(self, po): ''' Helper function to call _pkgExcludedRKNEVRA. Takes a package object. ''' return self._pkgExcludedRKT(po.repo, po.pkgKey, po.pkgtup) def addPackageExcluder(self, repoid, excluderid, excluder, *args): """ Add an "excluder" for all packages in the repo/sack. Can basically do anything based on nevra, changes lots of exclude decisions from "preload package; test; delPackage" into "load excluder". Excluderid is used so the caller doesn't have to track "have I loaded the excluder for this repo.", it's probably only useful when repoid is None ... if it turns out utterly worthless then it's still not a huge wart. """ if excluderid is not None and excluderid in self._pkgExcludeIds: return match = None regexp_match = None if False: pass elif excluder.endswith('.eq'): assert len(args) == 1 match = args[0].lower() elif excluder.endswith('.in'): assert len(args) == 1 match = args[0] elif excluder.endswith('.match'): assert len(args) == 1 match = args[0].lower() if misc.re_glob(match): regexp_match = re.compile(fnmatch.translate(match)).match elif excluder.endswith('.*'): assert len(args) == 0 elif excluder.endswith('.marked'): assert len(args) == 0 elif excluder.endswith('.washed'): assert len(args) == 0 # Really need to do this, need to cleanup pkgExcluder first though # or it does nothing. # self._pkgobjlist_dirty = True self._pkgExcluder.append((repoid, excluder, match, regexp_match)) if excluderid is not None: self._pkgExcludeIds[excluderid] = len(self._pkgExcluder) self._exclude_whitelist = set() self._pkgobjlist_dirty = True def _packageByKey(self, repo, pkgKey, exclude=True): """ Lookup a pkg by it's pkgKey, if we don't have it load it """ # Speed hack, so we don't load the pkg. if the pkgKey is dead. assert exclude if exclude and self._pkgKeyExcluded(repo, pkgKey): return None if repo not in self._key2pkg: self._key2pkg[repo] = {} self._pkgname2pkgkeys[repo] = {} if pkgKey not in self._key2pkg[repo]: sql = "SELECT pkgKey, pkgId, name, epoch, version, release, arch " \ "FROM packages WHERE pkgKey = ?" data = self._sql_MD('primary', repo, sql, (pkgKey,)).fetchone() if data is None: msg = "pkgKey %s doesn't exist in repo %s" % (pkgKey, repo) raise Errors.RepoError, msg if exclude and self._pkgExcludedRKD(repo, pkgKey, data): return None po = self.pc(repo, data) self._key2pkg[repo][pkgKey] = po self._pkgtup2pkgs.setdefault(po.pkgtup, []).append(po) pkgkeys = self._pkgname2pkgkeys[repo].setdefault(data['name'], []) pkgkeys.append(pkgKey) elif exclude and self._pkgExcluded(self._key2pkg[repo][pkgKey]): self._delPackageRK(repo, pkgKey) return None return self._key2pkg[repo][pkgKey] def _packageByKeyData(self, repo, pkgKey, data, exclude=True): """ Like _packageByKey() but we already have the data for .pc() """ assert exclude if exclude and self._pkgExcludedRKD(repo, pkgKey, data): return None if repo not in self._key2pkg: self._key2pkg[repo] = {} self._pkgname2pkgkeys[repo] = {} if data['pkgKey'] not in self._key2pkg.get(repo, {}): po = self.pc(repo, data) self._key2pkg[repo][pkgKey] = po self._pkgtup2pkgs.setdefault(po.pkgtup, []).append(po) pkgkeys = self._pkgname2pkgkeys[repo].setdefault(data['name'], []) pkgkeys.append(pkgKey) return self._key2pkg[repo][data['pkgKey']] def _pkgtupByKeyData(self, repo, pkgKey, data): """ Like _packageByKeyData() but we don't create the package, we just return the pkgtup. """ if self._pkgExcludedRKD(repo, pkgKey, data): return None prepo = self._key2pkg.get(repo) if prepo is None: self._key2pkg[repo] = {} self._pkgname2pkgkeys[repo] = {} elif data['pkgKey'] in prepo: return prepo[data['pkgKey']].pkgtup return (data['name'], data['arch'], data['epoch'], data['version'], data['release']) def _packagesByName(self, pkgname): """ Load all pkgnames from cache, with a given name. """ ret = [] for repo in self.primarydb: pkgkeys = self._pkgname2pkgkeys.get(repo, {}).get(pkgname, []) if not pkgkeys: continue for pkgkey in pkgkeys: pkg = self._packageByKey(repo, pkgkey) if pkg is None: continue ret.append(pkg) return ret def addDict(self, repo, datatype, dataobj, callback=None): if repo in self.added: if datatype in self.added[repo]: return else: self.added[repo] = [] if repo not in self.excludes: self.excludes[repo] = {} if dataobj is None: raise Errors.RepoError, "Tried to add None %s to %s" % (datatype, repo) if datatype == 'metadata': self.primarydb[repo] = dataobj elif datatype == 'filelists': self.filelistsdb[repo] = dataobj elif datatype == 'otherdata': self.otherdb[repo] = dataobj else: # We can not handle this yet... raise Errors.RepoError, "Sorry sqlite does not support %s in %s" % (datatype, repo) self.added[repo].append(datatype) # Get all files for a certain pkgId from the filelists.xml metadata # Search packages that either provide something containing name # or provide a file containing name def searchAll(self,name, query_type='like'): # this function is just silly and it reduces down to just this return self.searchPrco(name, 'provides') def _sql_pkgKey2po(self, repo, cur, pkgs=None, have_data=False): """ Takes a cursor and maps the pkgKey rows into a list of packages. """ if pkgs is None: pkgs = [] for ob in cur: if have_data: pkg = self._packageByKeyData(repo, ob['pkgKey'], ob) else: pkg = self._packageByKey(repo, ob['pkgKey']) if pkg is None: continue pkgs.append(pkg) return pkgs def _skip_all(self): """ Are we going to skip every package in all our repos? """ skip_all = True for repo in self.added: if repo not in self._all_excludes: skip_all = False break return skip_all @catchSqliteException def _search_primary_files(self, name): querytype = 'glob' name = os.path.normpath(name) if not misc.re_glob(name): querytype = '=' results = [] for (rep,cache) in self.primarydb.items(): if rep in self._all_excludes: continue cur = cache.cursor() executeSQL(cur, "select DISTINCT pkgKey from files where name %s ?" % querytype, (name,)) self._sql_pkgKey2po(rep, cur, results) return misc.unique(results) @catchSqliteException def _have_fastReturnFileEntries(self): """ Return true if pkg.returnFileEntries(primary_only=True) is fast. basically does "CREATE INDEX pkgfiles ON files (pkgKey);" exist. """ for (rep,cache) in self.primarydb.items(): if rep in self._all_excludes: continue cur = cache.cursor() executeSQL(cur, "PRAGMA index_info(pkgfiles)") # If we get anything, we're fine. There might be a better way of # saying "anything" but this works. for ob in cur: break else: return False return True def have_fastReturnFileEntries(self): """ Is calling pkg.returnFileEntries(primary_only=True) faster than using searchFiles(). """ if not hasattr(self, '_cached_fRFE'): self._cached_fRFE = self._have_fastReturnFileEntries() return self._cached_fRFE @catchSqliteException def searchFiles(self, name, strict=False): """search primary if file will be in there, if not, search filelists, use globs, if possible""" if self._skip_all(): return [] # optimizations: # if it is not glob, then see if it is in the primary.xml filelists, # if so, just use those for the lookup glob = True file_glob = True querytype = 'glob' name = os.path.normpath(name) dirname = os.path.dirname(name) filename = os.path.basename(name) if strict or not misc.re_glob(name): glob = False file_glob = False querytype = '=' elif not misc.re_glob(filename): file_glob = False # Take off the trailing slash to act like rpm if name[-1] == '/': name = name[:-1] pkgs = [] # ultra simple optimization if misc.re_primary_filename(name): return self._search_primary_files(name) if len(self.filelistsdb) == 0: # grab repo object from primarydb and force filelists population in this sack using repo # sack.populate(repo, mdtype, callback, cacheonly) for (repo,cache) in self.primarydb.items(): if repo in self._all_excludes: continue self.populate(repo, mdtype='filelists') # Check to make sure the DB data matches, this should always pass but # we've had weird errors. So check it for a bit. for repo in self.filelistsdb: # Only check each repo. once ... the libguestfs check :). if hasattr(repo, '_checked_filelists_pkgs'): continue pri_pkgs = self._sql_MD_pkg_num('primary', repo) fil_pkgs = self._sql_MD_pkg_num('filelists', repo) if pri_pkgs != fil_pkgs: raise Errors.RepoError repo._checked_filelists_pkgs = True sql_params = [] dirname_check = "" if not glob: (pattern, esc) = sql_esc(filename) dirname_check = "dirname = ? and filenames LIKE ? %s and " % esc sql_params.append(dirname) sql_params.append('%' + pattern + '%') elif not file_glob: (pattern, esc) = sql_esc(filename) dirname_check = "dirname GLOB ? and filenames LIKE ? %s and " % esc sql_params.append(dirname) sql_params.append('%' + pattern + '%') elif filename == '*': # We only care about matching on dirname... for (rep,cache) in self.filelistsdb.items(): if rep in self._all_excludes: continue cur = cache.cursor() sql_params.append(dirname) executeSQL(cur, """SELECT pkgKey FROM filelist WHERE dirname %s ?""" % (querytype,), sql_params) self._sql_pkgKey2po(rep, cur, pkgs) return misc.unique(pkgs) for (rep,cache) in self.filelistsdb.items(): if rep in self._all_excludes: continue cur = cache.cursor() # grab the entries that are a single file in the # filenames section, use sqlites globbing if it is a glob executeSQL(cur, "select pkgKey from filelist where \ %s length(filetypes) = 1 and \ dirname || ? || filenames \ %s ?" % (dirname_check, querytype), sql_params + ['/',name]) self._sql_pkgKey2po(rep, cur, pkgs) if file_glob: name_re = re.compile(fnmatch.translate(name)) def filelist_globber(sql_dirname, sql_filenames): # Note: Can't return bool, because sqlite doesn't like it in # weird ways. Test: # install '*bin/autoheader' # provides /lib/security/pam_loginuid.so files = sql_filenames.split('/') if not file_glob: return int(filename in files) fns = map(lambda f: '%s/%s' % (sql_dirname, f), files) for match in fns: if name_re.match(match): return 1 return 0 cache.create_function("filelist_globber", 2, filelist_globber) # for all the ones where filenames is multiple files, # make the files up whole and use python's globbing method executeSQL(cur, "select pkgKey from filelist where \ %s length(filetypes) > 1 \ and filelist_globber(dirname,filenames)" % dirname_check, sql_params) self._sql_pkgKey2po(rep, cur, pkgs) pkgs = misc.unique(pkgs) return pkgs @catchSqliteException def searchPrimaryFields(self, fields, searchstring): """search arbitrary fields from the primarydb for a string""" if self._skip_all(): return [] result = [] if len(fields) < 1: return result searchstring = searchstring.replace("'", "''") (searchstring, esc) = sql_esc(searchstring) sql = "select DISTINCT pkgKey from packages where %s like '%%%s%%'%s " % (fields[0], searchstring, esc) for f in fields[1:]: sql = "%s or %s like '%%%s%%'%s " % (sql, f, searchstring, esc) for (rep,cache) in self.primarydb.items(): cur = cache.cursor() executeSQL(cur, sql) self._sql_pkgKey2po(rep, cur, result) return result @catchSqliteException def searchPrimaryFieldsMultipleStrings(self, fields, searchstrings): """search arbitrary fields from the primarydb for a multiple strings return packages, number of items it matched as a list of tuples""" if self._skip_all(): return [] result = [] # (pkg, num matches) if not fields or not searchstrings: return result # NOTE: I can't see any reason not to use this all the time, speed # comparison shows them as basically equal. if len(searchstrings) > (constants.PATTERNS_MAX / len(fields)): tot = {} for searchstring in searchstrings: matches = self.searchPrimaryFields(fields, searchstring) for po in matches: tot[po] = tot.get(po, 0) + 1 for po in sorted(tot, key=operator.itemgetter, reverse=True): result.append((po, tot[po])) return result unionstring = "select pkgKey, SUM(cumul) AS total from ( " endunionstring = ")GROUP BY pkgKey ORDER BY total DESC" #SELECT pkgkey, SUM(cumul) AS total FROM (SELECT pkgkey, 1 #AS cumul FROM packages WHERE description LIKE '%foo%' UNION ... ) #GROUP BY pkgkey ORDER BY total DESC; selects = [] for s in searchstrings: s = s.replace("'", "''") (s, esc) = sql_esc(s) sql="select pkgKey,1 AS cumul from packages where %s like '%%%s%%'%s " % (fields[0], s, esc) for f in fields[1:]: sql = "%s or %s like '%%%s%%'%s " % (sql, f, s, esc) selects.append(sql) totalstring = unionstring + " UNION ALL ".join(selects) + endunionstring for (rep,cache) in self.primarydb.items(): cur = cache.cursor() executeSQL(cur, totalstring) for ob in cur: pkg = self._packageByKey(rep, ob['pkgKey']) if pkg is None: continue result.append((pkg, ob['total'])) return result @catchSqliteException def returnObsoletes(self, newest=False): if self._skip_all(): return {} if newest: raise NotImplementedError() obsoletes = {} for (rep,cache) in self.primarydb.items(): cur = cache.cursor() executeSQL(cur, "select packages.name as name,\ packages.pkgKey as pkgKey,\ packages.arch as arch, packages.epoch as epoch,\ packages.release as release, packages.version as version,\ obsoletes.name as oname, obsoletes.epoch as oepoch,\ obsoletes.release as orelease, obsoletes.version as oversion,\ obsoletes.flags as oflags\ from obsoletes,packages where obsoletes.pkgKey = packages.pkgKey") for ob in cur: key = ( _share_data(ob['name']), _share_data(ob['arch']), _share_data(ob['epoch']), _share_data(ob['version']), _share_data(ob['release'])) if self._pkgExcludedRKT(rep, ob['pkgKey'], key): continue (n,f,e,v,r) = ( _share_data(ob['oname']), _share_data(ob['oflags']), _share_data(ob['oepoch']), _share_data(ob['oversion']), _share_data(ob['orelease'])) key = _share_data(key) val = _share_data((n,f,(e,v,r))) obsoletes.setdefault(key,[]).append(val) return obsoletes @catchSqliteException def getPackageDetails(self,pkgId): for (rep,cache) in self.primarydb.items(): cur = cache.cursor() executeSQL(cur, "select * from packages where pkgId = ?", (pkgId,)) for ob in cur: return ob @catchSqliteException def _getListofPackageDetails(self, pkgId_list): pkgs = [] if len(pkgId_list) == 0: return pkgs pkgid_query = str(tuple(pkgId_list)) for (rep,cache) in self.primarydb.items(): cur = cache.cursor() executeSQL(cur, "select * from packages where pkgId in %s" %(pkgid_query,)) for ob in cur: pkgs.append(ob) return pkgs @catchSqliteException def _search_get_memoize(self, prcotype): if not hasattr(self, '_memoize_' + prcotype): memoize = {} for (rep,cache) in self.primarydb.items(): if rep in self._all_excludes: continue cur = cache.cursor() executeSQL(cur, "select * from %s" % prcotype) for x in cur: val = (_share_data(x['name']), _share_data(x['flags']), (_share_data(x['epoch']), _share_data(x['version']), _share_data(x['release']))) val = _share_data(val) key = (rep, val[0]) pkgkey = _share_data(x['pkgKey']) val = (pkgkey, val) memoize.setdefault(key, []).append(val) setattr(self, '_memoize_' + prcotype, memoize) return getattr(self, '_memoize_' + prcotype) @catchSqliteException def _search(self, prcotype, name, flags, version): if self._skip_all(): return {} name = to_unicode(name) if flags == 0: flags = None if type(version) in (str, type(None), unicode): req = (name, flags, rpmUtils.miscutils.stringToVersion( version)) elif type(version) in (tuple, list): # would this ever be a list? req = (name, flags, version) prcotype = _share_data(prcotype) req = _share_data(req) if req in self._search_cache[prcotype]: return self._search_cache[prcotype][req] result = { } # Requires is the biggest hit, pre-loading provides actually hurts # NOTE: Disabling atm. ... small install/updates get a significant hit. # And even large updates take a hit with the memoize path, maybe we # fixed something with later change? ... maybe I was on crack? # Speed seems to depend on _search_cache. if True: # prcotype != 'requires': primarydb_items = self.primarydb.items() preload = False else: primarydb_items = [] preload = True memoize = self._search_get_memoize(prcotype) for (rep,cache) in self.primarydb.items(): if rep in self._all_excludes: continue tmp = {} for x in memoize.get((rep, name), []): pkgkey, val = x if rpmUtils.miscutils.rangeCompare(req, val): tmp.setdefault(pkgkey, []).append(val) for pkgKey, hits in tmp.iteritems(): pkg = self._packageByKey(rep, pkgKey) if pkg is None: continue result[pkg] = hits for (rep,cache) in primarydb_items: if rep in self._all_excludes: continue cur = cache.cursor() executeSQL(cur, "select * from %s where name=?" % prcotype, (name,)) tmp = { } for x in cur: val = (_share_data(x['name']), _share_data(x['flags']), (_share_data(x['epoch']), _share_data(x['version']), _share_data(x['release']))) val = _share_data(val) if rpmUtils.miscutils.rangeCompare(req, val): tmp.setdefault(x['pkgKey'], []).append(val) for pkgKey, hits in tmp.iteritems(): pkg = self._packageByKey(rep, pkgKey) if pkg is None: continue result[pkg] = hits if prcotype != 'provides' or name[0] != '/': if not preload: self._search_cache[prcotype][req] = result return result if not misc.re_primary_filename(name): # If it is not in the primary.xml files # search the files.xml file info for pkg in self.searchFiles(name, strict=True): result[pkg] = [(name, None, None)] if not preload: self._search_cache[prcotype][req] = result return result # If it is a filename, search the primary.xml file info for pkg in self._search_primary_files(name): result[pkg] = [(name, None, None)] self._search_cache[prcotype][req] = result return result def getProvides(self, name, flags=None, version=(None, None, None)): return self._search("provides", name, flags, version) def getRequires(self, name, flags=None, version=(None, None, None)): return self._search("requires", name, flags, version) @catchSqliteException def searchNames(self, names=[], return_pkgtups=False): """return a list of packages matching any of the given names. This is only a match on package name, nothing else""" if self._skip_all(): return [] loaded_all_names = hasattr(self, 'pkgobjlist') returnList = [] user_names = set(names) names = [] for pkgname in user_names: if pkgname in self._pkgmatch_fails: continue if loaded_all_names or pkgname in self._pkgnames_loaded: returnList.extend(self._packagesByName(pkgname)) else: names.append(pkgname) if return_pkgtups: returnList = [pkg.pkgtup for pkg in returnList] if not names: return returnList max_entries = constants.PATTERNS_INDEXED_MAX if len(names) > max_entries: # Unique is done at user_names time, above. for names in seq_max_split(names, max_entries): returnList.extend(self.searchNames(names, return_pkgtups)) return returnList pat_sqls = [] qsql = """select pkgId,pkgKey,name,epoch,version,release,arch from packages where """ for name in names: pat_sqls.append("name = ?") qsql = qsql + " OR ".join(pat_sqls) for (repo, cache) in self.primarydb.items(): cur = cache.cursor() executeSQL(cur, qsql, names) if return_pkgtups: for ob in cur: pkgtup = self._pkgtupByKeyData(repo, ob['pkgKey'], ob) if pkgtup is None: continue returnList.append(pkgtup) continue self._sql_pkgKey2po(repo, cur, returnList, have_data=True) if not return_pkgtups: # Mark all the processed pkgnames as fully loaded self._pkgnames_loaded.update([name for name in names]) return returnList @catchSqliteException def searchPrco(self, name, prcotype): """return list of packages matching name and prcotype """ # we take name to be a string of some kind # we parse the string to see if it is a foo > 1.1 or if it is just 'foo' # or what - so we can answer correctly if self._skip_all(): return [] try: (n,f,(e,v,r)) = misc.string_to_prco_tuple(name) except Errors.MiscError, e: raise Errors.PackageSackError, to_unicode(e) # The _b means this is a byte string # The _u means this is a unicode string # A bare n is used when, it's unicode but hasn't been evaluated # whether that's actually the right thing to do n_b = n n_u = to_unicode(n) n = n_u glob = True querytype = 'glob' if not misc.re_glob(n): glob = False querytype = '=' basic_results = [] results = [] for (rep,cache) in self.primarydb.items(): cur = cache.cursor() executeSQL(cur, "select DISTINCT pkgKey from %s where name %s ?" % (prcotype,querytype), (n,)) self._sql_pkgKey2po(rep, cur, basic_results) # now we have a list of items matching just the name - let's match them out for po in basic_results: if misc.re_filename(n) and v is None: # file dep add all matches to the results results.append(po) continue if not glob: if po.checkPrco(prcotype, (n_b, f, (e,v,r))): results.append(po) else: # if it is a glob we can't really get any closer to checking it results.append(po) # If it's not a provides or a filename, we are done if prcotype != "provides": return results if not misc.re_filename(n): return results # If it is a filename, search the primary.xml file info results.extend(self._search_primary_files(n)) # If it is in the primary.xml files then skip the other check if misc.re_primary_filename(n) and not glob: return misc.unique(results) # If it is a filename, search the files.xml file info results.extend(self.searchFiles(n)) return misc.unique(results) def searchProvides(self, name): """return list of packages providing name (any evr and flag)""" if name in self._provmatch_fails: return [] ret = self.searchPrco(name, "provides") if not ret: self._provmatch_fails.add(name) return ret def searchRequires(self, name): """return list of packages requiring name (any evr and flag)""" return self.searchPrco(name, "requires") def searchObsoletes(self, name): """return list of packages obsoleting name (any evr and flag)""" return self.searchPrco(name, "obsoletes") def searchConflicts(self, name): """return list of packages conflicting with name (any evr and flag)""" return self.searchPrco(name, "conflicts") def db2class(self, db, nevra_only=False): print 'die die die die die db2class' class tmpObject: pass y = tmpObject() y.nevra = (db['name'],db['epoch'],db['version'],db['release'],db['arch']) y.sack = self y.pkgId = db['pkgId'] if nevra_only: return y y.hdrange = {'start': db['rpm_header_start'],'end': db['rpm_header_end']} y.location = {'href': db['location_href'],'value': '', 'base': db['location_base']} y.checksum = {'pkgid': 'YES','type': db['checksum_type'], 'value': db['pkgId'] } y.time = {'build': db['time_build'], 'file': db['time_file'] } y.size = {'package': db['size_package'], 'archive': db['size_archive'], 'installed': db['size_installed'] } y.info = {'summary': db['summary'], 'description': db['description'], 'packager': db['rpm_packager'], 'group': db['rpm_group'], 'buildhost': db['rpm_buildhost'], 'sourcerpm': db['rpm_sourcerpm'], 'url': db['url'], 'vendor': db['rpm_vendor'], 'license': db['rpm_license'] } return y @catchSqliteException def returnNewestByNameArch(self, naTup=None, patterns=None, ignore_case=False): # If naTup is set do it from the database otherwise use our parent's # returnNewestByNameArch if (not naTup): return yumRepo.YumPackageSack.returnNewestByNameArch(self, naTup, patterns, ignore_case) # First find all packages that fulfill naTup allpkg = [] for (rep,cache) in self.primarydb.items(): cur = cache.cursor() executeSQL(cur, "select pkgId,pkgKey,name,epoch,version,release,arch from packages where name=? and arch=?", naTup) self._sql_pkgKey2po(rep, cur, allpkg, have_data=True) # if we've got zilch then raise if not allpkg: raise Errors.PackageSackError, 'No Package Matching %s.%s' % naTup return misc.newestInList(allpkg) @catchSqliteException def returnNewestByName(self, name=None, patterns=None, ignore_case=False): """return list of newest packages based on name matching this means(in name.arch form): foo.i386 and foo.noarch will be compared to each other for highest version. Note that given: foo-1.i386; foo-2.i386 and foo-3.x86_64 The last _two_ pkgs will be returned, not just one of them. """ # If name is set do it from the database otherwise use our parent's # returnNewestByName if self._skip_all(): return [] if (not name): return yumRepo.YumPackageSack.returnNewestByName(self, name, patterns, ignore_case) # First find all packages that fulfill name allpkg = [] for (rep,cache) in self.primarydb.items(): cur = cache.cursor() executeSQL(cur, "select pkgId,pkgKey,name,epoch,version,release,arch from packages where name=?", (name,)) self._sql_pkgKey2po(rep, cur, allpkg, have_data=True) # if we've got zilch then raise if not allpkg: raise Errors.PackageSackError, 'No Package Matching %s' % name return misc.newestInList(allpkg) # Do what packages.matchPackageNames does, but query the DB directly @catchSqliteException def matchPackageNames(self, pkgspecs): if self._skip_all(): return [], [], [] matched = [] exactmatch = [] unmatched = list(pkgspecs) for p in pkgspecs: if misc.re_glob(p): query = PARSE_QUERY % ({ "op": "glob", "q": p }) matchres = matched else: query = PARSE_QUERY % ({ "op": "=", "q": p }) matchres = exactmatch for (rep, db) in self.primarydb.items(): cur = db.cursor() executeSQL(cur, query) pmatches = self._sql_pkgKey2po(rep, cur) if len(pmatches): unmatched.remove(p) matchres.extend(pmatches) exactmatch = misc.unique(exactmatch) matched = misc.unique(matched) unmatched = misc.unique(unmatched) return exactmatch, matched, unmatched def _setupPkgObjList(self, repoid=None, patterns=None, ignore_case=False): """Setup need_full and patterns for _yieldSQLDataList, also see if we can get away with just using searchNames(). """ if patterns is None: patterns = [] fields = ['name', 'sql_nameArch', 'sql_nameVerRelArch', 'sql_nameVer', 'sql_nameVerRel', 'sql_envra', 'sql_nevra'] need_full = False for pat in patterns: if (misc.re_full_search_needed(pat) and (ignore_case or pat not in self._pkgnames_loaded)): need_full = True break pat_max = constants.PATTERNS_MAX if not need_full: fields = ['name'] pat_max = constants.PATTERNS_INDEXED_MAX if len(patterns) > pat_max: patterns = [] if ignore_case: patterns = sql_esc_glob(patterns) else: tmp = [] need_glob = False for pat in patterns: if misc.re_glob(pat): tmp.append((pat, 'glob')) need_glob = True else: tmp.append((pat, '=')) if not need_full and not need_glob and patterns: return (need_full, patterns, fields, True) patterns = tmp return (need_full, patterns, fields, False) # @catchSqliteException has no effect on generators def _yieldSQLDataList(self, repoid, patterns, fields, ignore_case): """Yields all the package data for the given params. Excludes are done at this stage. """ pat_sqls = [] pat_data = [] for (pattern, rest) in patterns: if not ignore_case and pattern in self._pkgmatch_fails: continue for field in fields: if ignore_case: pat_sqls.append("%s LIKE ?%s" % (field, rest)) else: pat_sqls.append("%s %s ?" % (field, rest)) pat_data.append(pattern) if patterns and not pat_sqls: return if pat_sqls: qsql = _FULL_PARSE_QUERY_BEG + " OR ".join(pat_sqls) else: qsql = """select pkgId, pkgKey, name,epoch,version,release,arch from packages""" for (repo,cache) in self.primarydb.items(): if (repoid == None or repoid == repo.id): cur = cache.cursor() executeSQL(cur, qsql, pat_data) for x in cur: yield (repo, x) @catchSqliteException def _buildPkgObjList(self, repoid=None, patterns=None, ignore_case=False): """Builds a list of packages, only containing nevra information. Excludes are done at this stage. """ returnList = [] data = self._setupPkgObjList(repoid, patterns, ignore_case) (need_full, patterns, fields, names) = data if names: return self.searchNames(patterns) for (repo, x) in self._yieldSQLDataList(repoid, patterns, fields, ignore_case): # Can't use: _sql_pkgKey2po because we change repos. po = self._packageByKeyData(repo, x['pkgKey'], x) if po is None: continue returnList.append(po) if not patterns and repoid is None: self.pkgobjlist = returnList self._pkgnames_loaded = set() # Save memory if not need_full and repoid is None: # Mark all the processed pkgnames as fully loaded self._pkgnames_loaded.update([po.name for po in returnList]) if need_full: for (pat, rest) in patterns: if rest not in ('=', ''): # Wildcards: 'glob' or ' ESCAPE "!"' continue for pkg in returnList: if pkg.name == pat: self._pkgnames_loaded.add(pkg.name) break if not returnList: for (pat, rest) in patterns: self._pkgmatch_fails.add(pat) return returnList def returnPackages(self, repoid=None, patterns=None, ignore_case=False): """Returns a list of packages, only containing nevra information. The packages are processed for excludes. Note that the packages are always filtered to those matching the patterns/case. """ if self._skip_all(): return [] internal_pkgoblist = hasattr(self, 'pkgobjlist') if internal_pkgoblist: pkgobjlist = self._clean_pkgobjlist() else: pkgobjlist = self._buildPkgObjList(repoid, patterns, ignore_case) internal_pkgoblist = hasattr(self, 'pkgobjlist') if internal_pkgoblist and patterns: internal_pkgoblist = False pkgobjlist = parsePackages(pkgobjlist, patterns, not ignore_case, unique='repo-pkgkey') pkgobjlist = pkgobjlist[0] + pkgobjlist[1] # Can't unexclude things, and new excludes are done above... if repoid is None: if internal_pkgoblist: pkgobjlist = pkgobjlist[:] return pkgobjlist returnList = [] for po in pkgobjlist: if repoid != po.repoid: continue returnList.append(po) return returnList @catchSqliteException def simplePkgList(self, patterns=None, ignore_case=False): """Returns a list of pkg tuples (n, a, e, v, r), optionally from a single repoid. Note that the packages are always filtered to those matching the patterns/case. """ if self._skip_all(): return [] internal_pkgoblist = hasattr(self, 'pkgobjlist') if internal_pkgoblist: return yumRepo.YumPackageSack.simplePkgList(self, patterns, ignore_case) repoid = None returnList = [] # Haven't loaded everything, so _just_ get the pkgtups... data = self._setupPkgObjList(repoid, patterns, ignore_case) (need_full, patterns, fields, names) = data if names: return [pkg.pkgtup for pkg in self.searchNames(patterns)] for (repo, x) in self._yieldSQLDataList(repoid, patterns, fields, ignore_case): # NOTE: Can't unexclude things... pkgtup = self._pkgtupByKeyData(repo, x['pkgKey'], x) if pkgtup is None: continue returnList.append(pkgtup) return returnList @catchSqliteException def searchNevra(self, name=None, epoch=None, ver=None, rel=None, arch=None): """return list of pkgobjects matching the nevra requested""" if self._skip_all(): return [] returnList = [] if name: # Almost always true... for pkg in self.searchNames(names=[name]): match = True for (col, var) in [('epoch', epoch), ('version', ver), ('arch', arch), ('release', rel)]: if var and getattr(pkg, col) != var: match = False break if match: returnList.append(pkg) return returnList # make sure some dumbass didn't pass us NOTHING to search on empty = True for arg in (name, epoch, ver, rel, arch): if arg: empty = False if empty: return returnList # make up our execute string q = "select pkgId,pkgKey,name,epoch,version,release,arch from packages WHERE" for (col, var) in [('name', name), ('epoch', epoch), ('version', ver), ('arch', arch), ('release', rel)]: if var: if q[-5:] != 'WHERE': q = q + ' AND %s = "%s"' % (col, var) else: q = q + ' %s = "%s"' % (col, var) # Search all repositories for (rep,cache) in self.primarydb.items(): cur = cache.cursor() executeSQL(cur, q) self._sql_pkgKey2po(rep, cur, returnList, have_data=True) return returnList @catchSqliteException def excludeArchs(self, archlist): """excludes incompatible arches - archlist is a list of compat arches""" if self._arch_allowed is None: self._arch_allowed = set(archlist) else: self._arch_allowed = self._arch_allowed.intersection(archlist) sarchlist = map(lambda x: "'%s'" % x , archlist) arch_query = ",".join(sarchlist) for (rep, cache) in self.primarydb.items(): cur = cache.cursor() # This is a minor hack opt. for source repos. ... if they are # enabled normally, we don't want to exclude each package so we # check it and exclude the entire thing. if not rep.id.endswith("-source") or 'src' in self._arch_allowed: continue has_arch = False executeSQL(cur, "SELECT DISTINCT arch FROM packages") for row in cur: if row[0] in archlist: has_arch = True break if not has_arch: self._delAllPackages(rep) return # Simple helper functions # Return a string representing filenamelist (filenames can not contain /) def encodefilenamelist(filenamelist): return '/'.join(filenamelist) # Return a list representing filestring (filenames can not contain /) def decodefilenamelist(filenamestring): filenamestring = filenamestring.replace('//', '/') return filenamestring.split('/') # Return a string representing filetypeslist # filetypes should be file, dir or ghost def encodefiletypelist(filetypelist): result = '' ft2string = {'file': 'f','dir': 'd','ghost': 'g'} for x in filetypelist: result += ft2string[x] return result # Return a list representing filetypestring # filetypes should be file, dir or ghost def decodefiletypelist(filetypestring): string2ft = {'f':'file','d': 'dir','g': 'ghost'} return [string2ft[x] for x in filetypestring] # Query used by matchPackageNames # op is either '=' or 'like', q is the search term # Check against name, nameArch, nameVerRelArch, nameVer, nameVerRel, # envra, nevra PARSE_QUERY = """ select pkgKey from packages where name %(op)s '%(q)s' or name || '.' || arch %(op)s '%(q)s' or name || '-' || version %(op)s '%(q)s' or name || '-' || version || '-' || release %(op)s '%(q)s' or name || '-' || version || '-' || release || '.' || arch %(op)s '%(q)s' or epoch || ':' || name || '-' || version || '-' || release || '.' || arch %(op)s '%(q)s' or name || '-' || epoch || ':' || version || '-' || release || '.' || arch %(op)s '%(q)s' """ # This is roughly the same as above, and used by _buildPkgObjList(). # Use " to quote because we using ? ... and sqlutils.QmarkToPyformat gets # confused. _FULL_PARSE_QUERY_BEG = """ SELECT pkgId,pkgKey,name,epoch,version,release,arch, name || "." || arch AS sql_nameArch, name || "-" || version || "-" || release || "." || arch AS sql_nameVerRelArch, name || "-" || version AS sql_nameVer, name || "-" || version || "-" || release AS sql_nameVerRel, epoch || ":" || name || "-" || version || "-" || release || "." || arch AS sql_envra, name || "-" || epoch || ":" || version || "-" || release || "." || arch AS sql_nevra FROM packages WHERE """