1""" 2Manages the feed cache. 3 4@var iface_cache: A singleton cache object. You should normally use this rather than 5creating new cache objects. 6 7""" 8# Copyright (C) 2009, Thomas Leonard 9# See the README file for details, or visit http://0install.net. 10 11# Note: 12# 13# We need to know the modification time of each interface, because we refuse 14# to update to an older version (this prevents an attack where the attacker 15# sends back an old version which is correctly signed but has a known bug). 16# 17# The way we store this is a bit complicated due to backward compatibility: 18# 19# - GPG-signed interfaces have their signatures removed and a last-modified 20# attribute is stored containing the date from the signature. 21# 22# - XML-signed interfaces are stored unmodified with their signatures. The 23# date is extracted from the signature when needed. 24# 25# - Older versions used to add the last-modified attribute even to files 26# with XML signatures - these files therefore have invalid signatures and 27# we extract from the attribute for these. 28# 29# Eventually, support for the first and third cases will be removed. 30 31from__future__importprint_function 32 33importos,sys,time 34 35fromzeroinstallimport_,logger 36fromzeroinstall.supportimportbasedir,portable_rename,raise_with_traceback,unicode 37fromzeroinstall.injectorimportreader,model 38fromzeroinstall.injector.namespacesimportconfig_site,config_prog 39fromzeroinstall.injector.modelimportInterface,escape,unescape 40fromzeroinstallimportSafeException 41 42# If we started a check within this period, don't start another one: 43FAILED_CHECK_DELAY=60*60# 1 Hour
56"""A feed that has been downloaded but not yet added to the interface cache. 57 Feeds remain in this state until the user confirms that they trust at least 58 one of the signatures. 59 @ivar url: URL for the feed 60 @type url: str 61 @ivar signed_data: the untrusted data 62 @type signed_data: stream 63 @ivar sigs: signatures extracted from signed_data 64 @type sigs: [L{gpg.Signature}] 65 @ivar new_xml: the payload of the signed_data, or the whole thing if XML 66 @type new_xml: str 67 @since: 0.25""" 68__slots__=['url','signed_data','sigs','new_xml'] 69
71"""Downloaded data is a GPG-signed message. 72 @param url: the URL of the downloaded feed 73 @type url: str 74 @param signed_data: the downloaded data (not yet trusted) 75 @type signed_data: stream 76 @raise SafeException: if the data is not signed, and logs the actual data""" 77self.url=url 78self.signed_data=signed_data 79self.recheck()
82"""Download any required GPG keys not already on our keyring. 83 When all downloads are done (successful or otherwise), add any new keys 84 to the keyring, L{recheck}. 85 @param fetcher: fetcher to manage the download (was Handler before version 1.5) 86 @type fetcher: L{fetch.Fetcher} 87 @param key_mirror: URL of directory containing keys, or None to use feed's directory 88 @type key_mirror: str 89 @rtype: [L{zeroinstall.support.tasks.Blocker}]""" 90downloads={} 91blockers=[] 92forxinself.sigs: 93key_id=x.need_key() 94ifkey_id: 95try: 96importurlparse 97exceptImportError: 98fromurllibimportparseasurlparse# Python 3 99key_url=urlparse.urljoin(key_mirrororself.url,'%s.gpg'%key_id)100logger.info(_("Fetching key from %s"),key_url)101dl=fetcher.download_url(key_url,hint=feed_hint)102downloads[dl.downloaded]=(dl,dl.tempfile)103blockers.append(dl.downloaded)104105exception=None106any_success=False107108fromzeroinstall.supportimporttasks109110whileblockers:111yieldblockers112113old_blockers=blockers114blockers=[]115116forbinold_blockers:117dl,stream=downloads[b]118try:119tasks.check(b)120ifb.happened:121stream.seek(0)122self._downloaded_key(stream)123any_success=True124stream.close()125else:126blockers.append(b)127exceptException:128_type,exception,tb=sys.exc_info()129logger.warning(_("Failed to import key for '%(url)s': %(exception)s"),{'url':self.url,'exception':str(exception)})130stream.close()131132ifexceptionandnotany_success:133raise_with_traceback(exception,tb)134135self.recheck()
138"""@type stream: file"""139importshutil,tempfile140fromzeroinstall.injectorimportgpg141142logger.info(_("Importing key for feed '%s'"),self.url)143144# Python2.4: can't call fileno() on stream, so save to tmp file instead145tmpfile=tempfile.TemporaryFile(prefix='injector-dl-data-')146try:147shutil.copyfileobj(stream,tmpfile)148tmpfile.flush()149150tmpfile.seek(0)151gpg.import_key(tmpfile)152finally:153tmpfile.close()
156"""Set new_xml and sigs by reading signed_data.157 You need to call this when previously-missing keys are added to the GPG keyring."""158from.importgpg159try:160self.signed_data.seek(0)161stream,sigs=gpg.check_stream(self.signed_data)162assertsigs163164data=stream.read()165ifstreamisnotself.signed_data:166stream.close()167168self.new_xml=data169self.sigs=sigs170except:171self.signed_data.seek(0)172logger.info(_("Failed to check GPG signature. Data received was:\n")+repr(self.signed_data.read()))173raise
176"""177 The interface cache stores downloaded and verified interfaces in178 ~/.cache/0install.net/interfaces (by default).179180 There are methods to query the cache, add to it, check signatures, etc.181182 The cache is updated by L{fetch.Fetcher}.183184 Confusingly, this class is really two caches combined: the in-memory185 cache of L{model.Interface} objects, and an on-disk cache of L{model.ZeroInstallFeed}s.186 It will probably be split into two in future.187188 @ivar distro: the native distribution proxy189 @type distro: L{distro.Distribution}190191 @see: L{iface_cache} - the singleton IfaceCache instance.192 """193194__slots__=['_interfaces','_feeds','_distro','_config']195
197"""@param distro: distribution used to fetch "distribution:" feeds (since 0.49)198 @type distro: L{distro.Distribution}, or None to use the host distribution"""199self._interfaces={}200self._feeds={}201self._distro=distro
221"""Update a cached feed (using L{update_feed_from_network})222 if we trust the signatures.223 If we don't trust any of the signatures, do nothing.224 @param feed_url: the feed being updated225 @type feed_url: str226 @param sigs: signatures from L{gpg.check_stream}227 @type sigs: [L{gpg.Signature}]228 @param xml: the downloaded replacement feed document229 @type xml: str230 @type dry_run: bool231 @return: True if the feed was updated232 @rtype: bool233 @since: 0.48"""234from.importtrust235updated=self._oldest_trusted(sigs,trust.domain_from_url(feed_url))236ifupdatedisNone:returnFalse# None are trusted237238self.update_feed_from_network(feed_url,xml,updated,dry_run=dry_run)239returnTrue
247"""Update a cached feed.248 Called by L{update_feed_if_trusted} if we trust this data.249 After a successful update, L{writer} is used to update the feed's250 last_checked time.251 @param feed_url: the feed being updated252 @type feed_url: L{model.Interface}253 @param new_xml: the downloaded replacement feed document254 @type new_xml: str255 @param modified_time: the timestamp of the oldest trusted signature (used as an approximation to the feed's modification time)256 @type modified_time: long257 @type dry_run: bool258 @raises ReplayAttack: if modified_time is older than the currently cached time259 @since: 0.48"""260logger.debug(_("Updating '%(interface)s' from network; modified at %(time)s")%261{'interface':feed_url,'time':_pretty_time(modified_time)})262263self._import_new_feed(feed_url,new_xml,modified_time,dry_run)264265ifdry_run:return266267feed=self.get_feed(feed_url)268269from.importwriter270feed.last_checked=int(time.time())271writer.save_feed(feed)272273logger.info(_("Updated feed cache entry for %(interface)s (modified %(time)s)"),274{'interface':feed.get_name(),'time':_pretty_time(modified_time)})
277"""Write new_xml into the cache.278 @param feed_url: the URL for the feed being updated279 @type feed_url: str280 @param new_xml: the data to write281 @type new_xml: str282 @param modified_time: when new_xml was modified283 @type modified_time: int284 @type dry_run: bool285 @raises ReplayAttack: if the new mtime is older than the current one"""286assertmodified_time287assertisinstance(new_xml,bytes),repr(new_xml)288289upstream_dir=basedir.save_cache_path(config_site,'interfaces')290cached=os.path.join(upstream_dir,escape(feed_url))291292old_modified=None293ifos.path.exists(cached):294withopen(cached,'rb')asstream:295old_xml=stream.read()296ifold_xml==new_xml:297logger.debug(_("No change"))298# Update in-memory copy, in case someone else updated the disk copy299self.get_feed(feed_url,force=True)300return301old_modified=int(os.stat(cached).st_mtime)302303ifdry_run:304print(_("[dry-run] would cache feed {url} as {cached}").format(305url=feed_url,306cached=cached))307fromioimportBytesIO308fromzeroinstall.injectorimportqdom309root=qdom.parse(BytesIO(new_xml),filter_for_version=True)310feed=model.ZeroInstallFeed(root)311reader.update_user_feed_overrides(feed)312self._feeds[feed_url]=feed313return314315# Do we need to write this temporary file now?316try:317withopen(cached+'.new','wb')asstream:318stream.write(new_xml)319os.utime(cached+'.new',(modified_time,modified_time))320new_mtime=reader.check_readable(feed_url,cached+'.new')321assertnew_mtime==modified_time322323old_modified=self._get_signature_date(feed_url)orold_modified324325ifold_modified:326ifnew_mtime<old_modified:327raiseReplayAttack(_("New feed's modification time is "328"before old version!\nInterface: %(iface)s\nOld time: %(old_time)s\nNew time: %(new_time)s\n"329"Refusing update.")330%{'iface':feed_url,'old_time':_pretty_time(old_modified),'new_time':_pretty_time(new_mtime)})331ifnew_mtime==old_modified:332# You used to have to update the modification time manually.333# Now it comes from the signature, this check isn't useful334# and often causes problems when the stored format changes335# (e.g., when we stopped writing last-modified attributes)336pass337#raise SafeException("Interface has changed, but modification time "338# "hasn't! Refusing update.")339except:340os.unlink(cached+'.new')341raise342343portable_rename(cached+'.new',cached)344logger.debug(_("Saved as %s")%cached)345346self.get_feed(feed_url,force=True)
349"""Get a feed from the cache.350 @param url: the URL of the feed351 @type url: str352 @param force: load the file from disk again353 @type force: bool354 @param selections_ok: if url is a local selections file, return that instead355 @type selections_ok: bool356 @return: the feed, or None if it isn't cached357 @rtype: L{model.ZeroInstallFeed}"""358ifnotforce:359feed=self._feeds.get(url,False)360iffeed!=False:361returnfeed362363ifurl.startswith('distribution:'):364master_feed=self.get_feed(url.split(':',1)[1])365ifnotmaster_feed:366returnNone# Can't happen?367feed=self.distro.get_feed(master_feed)368else:369feed=reader.load_feed_from_cache(url,selections_ok=selections_ok)370ifselections_okandfeedandnotisinstance(feed,model.ZeroInstallFeed):371assertfeed.selectionsisnotNone372returnfeed# (it's actually a selections document)373iffeed:374reader.update_user_feed_overrides(feed)375self._feeds[url]=feed376returnfeed
379"""Get the interface for uri, creating a new one if required.380 New interfaces are initialised from the disk cache, but not from381 the network.382 @param uri: the URI of the interface to find383 @type uri: str384 @rtype: L{model.Interface}"""385iftype(uri)==str:386uri=unicode(uri)387assertisinstance(uri,unicode)388389ifuriinself._interfaces:390returnself._interfaces[uri]391392logger.debug(_("Initialising new interface object for %s"),uri)393self._interfaces[uri]=Interface(uri)394reader.update_from_cache(self._interfaces[uri],iface_cache=self)395returnself._interfaces[uri]
398"""List all interfaces in the cache.399 @rtype: [str]"""400all=set()401fordinbasedir.load_cache_paths(config_site,'interfaces'):402forleafinos.listdir(d):403ifnotleaf.startswith('.'):404all.add(unescape(leaf))405returnlist(all)# Why not just return the set?
408"""Get the path of a cached icon for an interface.409 @param iface: interface whose icon we want410 @type iface: L{Interface}411 @return: the path of the cached icon, or None if not cached.412 @rtype: str"""413returnbasedir.load_first_cache(config_site,'interface_icons',414escape(iface.uri))
417"""Verify the cached interface using GPG.418 Only new-style XML-signed interfaces retain their signatures in the cache.419 @param uri: the feed to check420 @type uri: str421 @return: a list of signatures, or None422 @rtype: [L{gpg.Signature}] or None423 @since: 0.25"""424from.importgpg425ifos.path.isabs(uri):426old_iface=uri427else:428old_iface=basedir.load_first_cache(config_site,'interfaces',escape(uri))429ifold_ifaceisNone:430returnNone431try:432withopen(old_iface,'rb')asstream:433returngpg.check_stream(stream)[1]434exceptSafeExceptionasex:435logger.info(_("No signatures (old-style interface): %s")%ex)436returnNone
439"""Read the date-stamp from the signature of the cached interface.440 If the date-stamp is unavailable, returns None.441 @type uri: str442 @rtype: int"""443from.importtrust444sigs=self.get_cached_signatures(uri)445ifsigs:446returnself._oldest_trusted(sigs,trust.domain_from_url(uri))
449"""Return the date of the oldest trusted signature in the list, or None if there450 are no trusted sigs in the list.451 @type sigs: [L{zeroinstall.injector.gpg.ValidSig}]452 @type domain: str453 @rtype: int"""454trusted=[s.get_timestamp()forsinsigsifs.is_trusted(domain)]455iftrusted:456returnmin(trusted)457returnNone
460"""Touch a 'last-check-attempt' timestamp file for this feed.461 If url is a local path, nothing happens.462 This prevents us from repeatedly trying to download a failing feed many463 times in a short period.464 @type url: str"""465ifos.path.isabs(url):466return467feeds_dir=basedir.save_cache_path(config_site,config_prog,'last-check-attempt')468timestamp_path=os.path.join(feeds_dir,model._pretty_escape(url))469fd=os.open(timestamp_path,os.O_WRONLY|os.O_CREAT,0o644)470os.close(fd)471os.utime(timestamp_path,None)# In case file already exists
474"""Return the time of the most recent update attempt for a feed.475 @type url: str476 @return: The time, or None if none is recorded477 @rtype: float | None478 @see: L{mark_as_checking}"""479timestamp_path=basedir.load_first_cache(config_site,config_prog,'last-check-attempt',model._pretty_escape(url))480iftimestamp_path:481returnos.stat(timestamp_path).st_mtime482returnNone
485"""Get all feeds that add to this interface.486 This is the feeds explicitly added by the user, feeds added by the distribution,487 and feeds imported by a <feed> in the main feed (but not recursively, at present).488 @type iface: L{Interface}489 @rtype: L{Feed}490 @since: 0.48"""491main_feed=self.get_feed(iface.uri)492ifmain_feed:493returniface.extra_feeds+main_feed.feeds494else:495returniface.extra_feeds
498"""Get all feeds for this interface. This is a mapping from feed URLs499 to ZeroInstallFeeds. It includes the interface's main feed, plus the500 resolution of every feed returned by L{get_feed_imports}. Uncached501 feeds are indicated by a value of None.502 @type iface: L{Interface}503 @rtype: {str: L{ZeroInstallFeed} | None}504 @since: 0.48"""505main_feed=self.get_feed(iface.uri)506results={iface.uri:main_feed}507forimpiniface.extra_feeds:508try:509results[imp.uri]=self.get_feed(imp.uri)510exceptSafeExceptionasex:511logger.warning("Failed to load feed '%s: %s",imp.uri,ex)512ifmain_feed:513forimpinmain_feed.feeds:514results[imp.uri]=self.get_feed(imp.uri)515returnresults
518"""Return all implementations from all of iface's feeds.519 @type iface: L{Interface}520 @rtype: [L{Implementation}]521 @since: 0.48"""522impls=[]523forfeedinself.get_feeds(iface).values():524iffeed:525impls+=feed.implementations.values()526returnimpls
529"""Return a list of Interfaces for which feed can be a feed.530 This is used by B{0install add-feed}.531 @param feed: the feed532 @type feed: L{model.ZeroInstallFeed} (or, deprecated, a URL)533 @rtype: [model.Interface]534 @raise SafeException: If there are no known feeds.535 @since: 0.53"""536537ifnotisinstance(feed,model.ZeroInstallFeed):538# (deprecated)539feed=self.get_feed(feed)540iffeedisNone:541raiseSafeException("Feed is not cached and using deprecated API")542543ifnotfeed.feed_for:544raiseSafeException(_("Missing <feed-for> element in '%s'; "545"it can't be used as a feed for any other interface.")%feed.url)546feed_targets=feed.feed_for547logger.debug(_("Feed targets: %s"),feed_targets)548return[self.get_interface(uri)foruriinfeed_targets]
551"""Check whether feed needs updating, based on the configured L{config.Config.freshness}.552 None is considered to be stale.553 If we already tried to update the feed within FAILED_CHECK_DELAY, returns false.554 @type feed_url: str555 @type freshness_threshold: int556 @return: True if feed should be updated557 @rtype: bool558 @since: 0.53"""559ifisinstance(feed_url,model.ZeroInstallFeed):560feed_url=feed_url.url# old API561eliffeed_urlisNone:562returnTrue# old API563564now=time.time()565566feed=self.get_feed(feed_url)567iffeedisnotNone:568iffeed.local_pathisnotNone:569returnFalse# Local feeds are never stale570571iffeed.last_modifiedisnotNone:572staleness=now-(feed.last_checkedor0)573logger.debug(_("Staleness for %(feed)s is %(staleness).2f hours"),{'feed':feed,'staleness':staleness/3600.0})574575iffreshness_threshold<=0orstaleness<freshness_threshold:576returnFalse# Fresh enough for us577# else we've never had it578579last_check_attempt=self.get_last_check_attempt(feed_url)580iflast_check_attemptandlast_check_attempt>now-FAILED_CHECK_DELAY:581logger.debug(_("Stale, but tried to check recently (%s) so not rechecking now."),time.ctime(last_check_attempt))582returnFalse583584returnTrue
587"""Generator for C{iface.feeds} that are valid for this architecture.588 @type iface: L{model.Interface}589 @rtype: generator590 @see: L{arch}591 @since: 0.53"""592forfinself.get_feed_imports(iface):593iff.osinarch.os_ranksandf.machineinarch.machine_ranks:594yieldf595else:596logger.debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),597{'feed':f,'os':f.os,'machine':f.machine})