40"""Fetches information about a GPG key from a key-info server. 41 See L{Fetcher.fetch_key_info} for details. 42 @since: 0.42 43 44 Example: 45 46 >>> kf = KeyInfoFetcher(fetcher, 'https://server', fingerprint) 47 >>> while True: 48 print kf.info 49 if kf.blocker is None: break 50 print kf.status 51 yield kf.blocker 52 """
92"""Downloads and stores various things. 93 @ivar config: used to get handler, iface_cache and stores 94 @type config: L{config.Config} 95 @ivar key_info: caches information about GPG keys 96 @type key_info: {str: L{KeyInfoFetcher}} 97 """ 98__slots__=['config','key_info','_scheduler','external_store'] 99
122"""Follow a Recipe.123 @type required_digest: str124 @type recipe: L{Recipe}125 @type stores: L{zeroinstall.zerostore.Stores}126 @type force: bool127 @param impl_hint: the Implementation this is for (if any) as a hint for the GUI128 @type dry_run: bool129 @type may_use_mirror: bool130 @see: L{download_impl} uses this method when appropriate"""131# Maybe we're taking this metaphor too far?132133# Start a download for each ingredient134blockers=[]135steps=[]136try:137forstepdatainrecipe.steps:138cls=StepRunner.class_for(stepdata)139step=cls(stepdata,impl_hint=impl_hint,may_use_mirror=may_use_mirror)140step.prepare(self,blockers)141steps.append(step)142143whileblockers:144yieldblockers145tasks.check(blockers)146blockers=[bforbinblockersifnotb.happened]147148ifself.external_store:149# Note: external_store will not yet work with non-<archive> steps.150streams=[step.streamforstepinsteps]151self._add_to_external_store(required_digest,recipe.steps,streams)152else:153# Create an empty directory for the new implementation154store=stores.stores[0]155tmpdir=store.get_tmp_dir_for(required_digest)156try:157# Unpack each of the downloaded archives into it in turn158forstepinsteps:159step.apply(tmpdir)160# Check that the result is correct and store it in the cache161store.check_manifest_and_rename(required_digest,tmpdir,dry_run=dry_run)162tmpdir=None163finally:164# If unpacking fails, remove the temporary directory165iftmpdirisnotNone:166support.ro_rmtree(tmpdir)167finally:168forstepinsteps:169try:170step.close()171exceptIOErrorasex:172# Can get "close() called during173# concurrent operation on the same file174# object." if we're unlucky (Python problem).175logger.info("Failed to close: %s",ex)
178"""Return the URL of a mirror for this feed.179 @type feed_url: str180 @type resource: str181 @rtype: str"""182ifself.config.mirrorisNone:183returnNone184iffeed_url.startswith('http://')orfeed_url.startswith('https://'):185ifsupport.urlparse(feed_url).hostname=='localhost':186returnNone187return'%s/%s/%s'%(self.config.mirror,_get_feed_dir(feed_url),resource)188returnNone
233"""Download the feed, download any required keys, confirm trust if needed and import.234 @param feed_url: the feed to be downloaded235 @type feed_url: str236 @param iface_cache: (deprecated)237 @type iface_cache: L{zeroinstall.injector.iface_cache.IfaceCache} | None238 @rtype: L{zeroinstall.support.tasks.Blocker}"""239from.downloadimportDownloadAborted240241assertiface_cacheisNoneoriface_cacheisself.config.iface_cache242243ifnotself.config.handler.dry_run:244try:245self.config.iface_cache.mark_as_checking(feed_url)246exceptOSErrorasex:247retval=tasks.Blocker("mark_as_checking")248retval.trigger(exception=(ex,None))249returnretval250251logger.debug(_("download_and_import_feed %(url)s"),{'url':feed_url})252assertnotos.path.isabs(feed_url)253254iffeed_url.startswith('distribution:'):255returnself.get_packagekit_feed(feed_url)256257primary=self._download_and_import_feed(feed_url,use_mirror=False)258259@tasks.named_async("monitor feed downloads for "+feed_url)260defwait_for_downloads(primary):261# Download just the upstream feed, unless it takes too long...262timeout=tasks.TimeoutBlocker(5,'Mirror timeout')# 5 seconds263264yieldprimary,timeout265tasks.check(timeout)266267try:268tasks.check(primary)269ifprimary.happened:270return# OK, primary succeeded!271# OK, maybe it's just being slow...272logger.info("Feed download from %s is taking a long time.",feed_url)273primary_ex=None274exceptNoTrustedKeysasex:275raise# Don't bother trying the mirror if we have a trust problem276exceptReplayAttackasex:277raise# Don't bother trying the mirror if we have a replay attack278exceptDownloadAbortedasex:279raise# Don't bother trying the mirror if the user cancelled280exceptSafeExceptionasex:281# Primary failed282primary=None283primary_ex=ex284logger.warning(_("Feed download from %(url)s failed: %(exception)s"),{'url':feed_url,'exception':ex})285286# Start downloading from mirror...287mirror=self._download_and_import_feed(feed_url,use_mirror=True)288289# Wait until both mirror and primary tasks are complete...290whileTrue:291blockers=list(filter(None,[primary,mirror]))292ifnotblockers:293break294yieldblockers295296ifprimary:297try:298tasks.check(primary)299ifprimary.happened:300primary=None301# No point carrying on with the mirror once the primary has succeeded302ifmirror:303logger.info(_("Primary feed download succeeded; aborting mirror download for %s")%feed_url)304mirror.dl.abort()305exceptSafeExceptionasex:306primary=None307primary_ex=ex308logger.info(_("Feed download from %(url)s failed; still trying mirror: %(exception)s"),{'url':feed_url,'exception':ex})309310ifmirror:311try:312tasks.check(mirror)313ifmirror.happened:314mirror=None315ifprimary_ex:316# We already warned; no need to raise an exception too,317# as the mirror download succeeded.318primary_ex=None319exceptReplayAttackasex:320logger.info(_("Version from mirror is older than cached version; ignoring it: %s"),ex)321mirror=None322primary_ex=None323exceptSafeExceptionasex:324logger.info(_("Mirror download failed: %s"),ex)325mirror=None326327ifprimary_ex:328raiseprimary_ex
333"""Download and import a feed.334 @type feed_url: str335 @param use_mirror: False to use primary location; True to use mirror.336 @type use_mirror: bool337 @rtype: L{zeroinstall.support.tasks.Blocker}"""338ifuse_mirror:339url=self.get_feed_mirror(feed_url)340ifurlisNone:returnNone341logger.info(_("Trying mirror server for feed %s")%feed_url)342else:343url=feed_url344345ifself.config.handler.dry_run:346print(_("[dry-run] downloading feed {url}").format(url=url))347dl=self.download_url(url,hint=feed_url)348stream=dl.tempfile349350@tasks.named_async("fetch_feed "+url)351deffetch_feed():352try:353yielddl.downloaded354tasks.check(dl.downloaded)355356pending=PendingFeed(feed_url,stream)357358ifuse_mirror:359# If we got the feed from a mirror, get the key from there too360key_mirror=self.config.mirror+'/keys/'361else:362key_mirror=None363364keys_downloaded=tasks.Task(pending.download_keys(self,feed_hint=feed_url,key_mirror=key_mirror),_("download keys for %s")%feed_url)365yieldkeys_downloaded.finished366tasks.check(keys_downloaded.finished)367368dry_run=self.handler.dry_run369ifnotself.config.iface_cache.update_feed_if_trusted(pending.url,pending.sigs,pending.new_xml,dry_run=dry_run):370blocker=self.config.trust_mgr.confirm_keys(pending)371ifblocker:372yieldblocker373tasks.check(blocker)374ifnotself.config.iface_cache.update_feed_if_trusted(pending.url,pending.sigs,pending.new_xml,dry_run=dry_run):375raiseNoTrustedKeys(_("No signing keys trusted; not importing"))376finally:377stream.close()
399"""Download an implementation.400 @param impl: the selected implementation401 @type impl: L{model.ZeroInstallImplementation}402 @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)403 @type retrieval_method: L{model.RetrievalMethod}404 @param stores: where to store the downloaded implementation405 @type stores: L{zerostore.Stores}406 @type force: bool407 @rtype: L{tasks.Blocker}"""408assertimpl409assertretrieval_method410411ifisinstance(retrieval_method,DistributionSource):412returnretrieval_method.install(self.handler)413414fromzeroinstall.zerostoreimportmanifest,parse_algorithm_digest_pair415best=None416fordigestinimpl.digests:417alg_name,digest_value=parse_algorithm_digest_pair(digest)418alg=manifest.algorithms.get(alg_name,None)419ifalgand(bestisNoneorbest.rating<alg.rating):420best=alg421required_digest=digest422423ifbestisNone:424ifnotimpl.digests:425raiseSafeException(_("No <manifest-digest> given for '%(implementation)s' version %(version)s")%426{'implementation':impl.feed.get_name(),'version':impl.get_version()})427raiseSafeException(_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s")%428{'algorithms':impl.digests,'implementation':impl.feed.get_name(),'version':impl.get_version()})429430@tasks.async431defdownload_impl(method):432original_exception=None433whileTrue:434ifnotisinstance(method,Recipe):435# turn an individual method into a single-step Recipe436step=method437method=Recipe()438method.steps.append(step)439440try:441blocker=self.cook(required_digest,method,stores,442impl_hint=impl,443dry_run=self.handler.dry_run,444may_use_mirror=original_exceptionisNone)445yieldblocker446tasks.check(blocker)447exceptdownload.DownloadErrorasex:448iforiginal_exception:449logger.info("Error from mirror: %s",ex)450raiseoriginal_exception451else:452original_exception=ex453mirror_url=self._get_impl_mirror(impl)454ifmirror_urlisnotNone:455logger.info("%s: trying implementation mirror at %s",ex,mirror_url)456method=model.DownloadSource(impl,mirror_url,457None,None,type='application/x-bzip-compressed-tar')458continue# Retry459raise460exceptSafeExceptionasex:461raiseSafeException("Error fetching {url} {version}: {ex}".format(462url=impl.feed.url,463version=impl.get_version(),464ex=ex))465break466467self.handler.impl_added_to_store(impl)
471"""@type required_digest: str"""472fromzeroinstall.zerostore.unpackimporttype_from_url473474# combine archive path, extract directory and MIME type arguments in an alternating fashion475paths=map(lambdastream:stream.name,streams)476extracts=map(lambdastep:step.extractor"",steps)477types=map(lambdastep:step.typeortype_from_url(step.url),steps)478args=[None]*(len(paths)+len(extracts)+len(types))479args[::3]=paths480args[1::3]=extracts481args[2::3]=types482483# close file handles to allow external processes access484forstreaminstreams:485stream.close()486487# delegate extracting archives to external tool488importsubprocess489subprocess.call([self.external_store,"add",required_digest]+args)490491# delete temp files492forpathinpaths:493os.remove(path)
497"""Fetch an archive. You should normally call L{download_impl}498 instead, since it handles other kinds of retrieval method too.499 It is the caller's responsibility to ensure that the returned stream is closed.500 @type download_source: L{DownloadSource}501 @type force: bool502 @type may_use_mirror: bool503 @rtype: (L{Blocker}, file)"""504fromzeroinstall.zerostoreimportunpack505506mime_type=download_source.type507ifnotmime_type:508mime_type=unpack.type_from_url(download_source.url)509ifnotmime_type:510raiseSafeException(_("No 'type' attribute on archive, and I can't guess from the name (%s)")%download_source.url)511ifnotself.external_store:512unpack.check_type_ok(mime_type)513514ifmay_use_mirror:515mirror=self._get_archive_mirror(download_source)516else:517mirror=None518519ifself.config.handler.dry_run:520print(_("[dry-run] downloading archive {url}").format(url=download_source.url))521dl=self.download_url(download_source.url,hint=impl_hint,mirror_url=mirror)522ifdownload_source.sizeisnotNone:523dl.expected_size=download_source.size+(download_source.start_offsetor0)524# (else don't know sizes for mirrored archives)525return(dl.downloaded,dl.tempfile)
528"""Fetch a single file. You should normally call L{download_impl}529 instead, since it handles other kinds of retrieval method too.530 It is the caller's responsibility to ensure that the returned stream is closed.531 @type download_source: L{zeroinstall.injector.model.FileSource}532 @type impl_hint: L{zeroinstall.injector.model.ZeroInstallImplementation} | None533 @rtype: tuple"""534ifself.config.handler.dry_run:535print(_("[dry-run] downloading file {url}").format(url=download_source.url))536537dl=self.download_url(download_source.url,hint=impl_hint)538dl.expected_size=download_source.size539return(dl.downloaded,dl.tempfile)
543"""Download an icon for this interface and add it to the544 icon cache. If the interface has no icon do nothing.545 @type interface: L{zeroinstall.injector.model.Interface}546 @type force: bool547 @return: the task doing the import, or None548 @rtype: L{tasks.Task}"""549logger.debug("download_icon %(interface)s",{'interface':interface})550551modification_time=None552existing_icon=self.config.iface_cache.get_icon_path(interface)553ifexisting_icon:554file_mtime=os.stat(existing_icon).st_mtime555fromemail.utilsimportformatdate556modification_time=formatdate(timeval=file_mtime,localtime=False,usegmt=True)557558feed=self.config.iface_cache.get_feed(interface.uri)559iffeedisNone:560returnNone561562# Find a suitable icon to download563foriconinfeed.get_metadata(XMLNS_IFACE,'icon'):564type=icon.getAttribute('type')565iftype!='image/png':566logger.debug(_('Skipping non-PNG icon'))567continue568source=icon.getAttribute('href')569ifsource:570break571logger.warning(_('Missing "href" attribute on <icon> in %s'),interface)572else:573logger.info(_('No PNG icons found in %s'),interface)574return575576dl=self.download_url(source,hint=interface,modification_time=modification_time)577578@tasks.async579defdownload_and_add_icon():580stream=dl.tempfile581try:582yielddl.downloaded583tasks.check(dl.downloaded)584ifdl.unmodified:return585stream.seek(0)586587importshutil,tempfile588icons_cache=basedir.save_cache_path(config_site,'interface_icons')589590tmp_file=tempfile.NamedTemporaryFile(dir=icons_cache,delete=False)591shutil.copyfileobj(stream,tmp_file)592tmp_file.close()593594icon_file=os.path.join(icons_cache,escape(interface.uri))595portable_rename(tmp_file.name,icon_file)596finally:597stream.close()
602"""Download the given implementations, choosing a suitable retrieval method for each.603 If any of the retrieval methods are DistributionSources and604 need confirmation, handler.confirm is called to check that the605 installation should proceed.606 @type implementations: [L{zeroinstall.injector.model.ZeroInstallImplementation}]607 @type stores: L{zeroinstall.zerostore.Stores}608 @rtype: L{zeroinstall.support.tasks.Blocker}"""609unsafe_impls=[]610611to_download=[]612forimplinimplementations:613logger.debug(_("start_downloading_impls: for %(feed)s get %(implementation)s"),{'feed':impl.feed,'implementation':impl})614source=self.get_best_source(impl)615ifnotsource:616raiseSafeException(_("Implementation %(implementation_id)s of interface %(interface)s"617" cannot be downloaded (no download locations given in "618"interface!)")%{'implementation_id':impl.id,'interface':impl.feed.get_name()})619to_download.append((impl,source))620621ifisinstance(source,DistributionSource)andsource.needs_confirmation:622unsafe_impls.append(source.package_id)623624@tasks.async625defdownload_impls():626ifunsafe_impls:627confirm=self.handler.confirm_install(_('The following components need to be installed using native packages. '628'These come from your distribution, and should therefore be trustworthy, but they also '629'run with extra privileges. In particular, installing them may run extra services on your '630'computer or affect other users. You may be asked to enter a password to confirm. The '631'packages are:\n\n')+('\n'.join('- '+xforxinunsafe_impls)))632yieldconfirm633tasks.check(confirm)634635blockers=[]636637forimpl,sourceinto_download:638blockers.append(self.download_impl(impl,source,stores))639640# Record the first error log the rest641error=[]642defdl_error(ex,tb=None):643iferror:644self.handler.report_error(ex)645else:646error.append((ex,tb))
662"""Return the best download source for this implementation.663 @type impl: L{zeroinstall.injector.model.ZeroInstallImplementation}664 @rtype: L{model.RetrievalMethod}"""665ifimpl.download_sources:666returnimpl.download_sources[0]667returnNone
670"""The most low-level method here; just download a raw URL.671 It is the caller's responsibility to ensure that dl.stream is closed.672 @param url: the location to download from673 @type url: str674 @param hint: user-defined data to store on the Download (e.g. used by the GUI)675 @param modification_time: don't download unless newer than this676 @param mirror_url: an altertive URL to try if this one fails677 @type mirror_url: str678 @rtype: L{download.Download}679 @since: 1.5"""680ifnot(url.startswith('http:')orurl.startswith('https:')orurl.startswith('ftp:')):681raiseSafeException(_("Unknown scheme in download URL '%s'")%url)682dl=download.Download(url,hint=hint,modification_time=modification_time,expected_size=expected_size,auto_delete=notself.external_store)683dl.mirror=mirror_url684self.handler.monitor_download(dl)685dl.downloaded=self.scheduler.download(dl)686returndl
706"""@type model: L{zeroinstall.injector.model.RetrievalMethod}"""707forsubclsincls.__subclasses__():708ifsubcls.model_type==type(model):709returnsubcls710raiseException(_("Unknown download type for '%s'")%model)
795"""Takes a cross-platform relative path (i.e using forward slashes, even on windows)796 and returns the absolute, platform-native version of the path.797 If the path does not resolve to a location within `base`, a SafeError is raised.798 @type base: str799 @type crossplatform_path: str800 @rtype: str801 @since: 1.10"""802assertos.path.isabs(base)803ifcrossplatform_path.startswith("/"):804raiseSafeException("path %r is not within the base directory"%(crossplatform_path,))805native_path=os.path.join(*crossplatform_path.split("/"))806fullpath=os.path.realpath(os.path.join(base,native_path))807base=os.path.realpath(base)808ifnotfullpath.startswith(base+os.path.sep):809raiseSafeException("path %r is not within the base directory"%(crossplatform_path,))810returnfullpath