1 2"""Processing of implementation manifests. 3 4A manifest is a string representing a directory tree, with the property 5that two trees will generate identical manifest strings if and only if: 6 7 - They have extactly the same set of files, directories and symlinks. 8 - For each pair of corresponding directories in the two sets: 9 - The mtimes are the same (OldSHA1 only). 10 - For each pair of corresponding files in the two sets: 11 - The size, executable flag and mtime are the same. 12 - The contents have matching secure hash values. 13 - For each pair of corresponding symlinks in the two sets: 14 - The mtime and size are the same. 15 - The targets have matching secure hash values. 16 17The manifest is typically processed with a secure hash itself. So, the idea is that 18any significant change to the contents of the tree will change the secure hash value 19of the manifest. 20 21A top-level ".manifest" file is ignored. 22""" 23 24# Copyright (C) 2009, Thomas Leonard 25# See the README file for details, or visit http://0install.net. 26 27 28importos,stat,base64 29fromzeroinstallimportSafeException,_,logger 30fromzeroinstall.zerostoreimportBadDigest,parse_algorithm_digest_pair,format_algorithm_digest_pair 31 32importhashlib 33sha1_new=hashlib.sha1 34
36"""Abstract base class for algorithms. 37 An algorithm knows how to generate a manifest from a directory tree. 38 @ivar rating: how much we like this algorithm (higher is better) 39 @type rating: int 40 """
47"""Create a new digest. Call update() on the returned object to digest the data. 48 Call getID() to turn it into a full ID string.""" 49raiseException('Abstract')
61"""@type root: str""" 62defrecurse(sub): 63# To ensure that a line-by-line comparison of the manifests 64# is possible, we require that filenames don't contain newlines. 65# Otherwise, you can name a file so that the part after the \n 66# would be interpreted as another line in the manifest. 67if'\n'insub:raiseBadDigest("Newline in filename '%s'"%sub) 68assertsub.startswith('/') 69 70ifsub=='/.manifest':return 71 72full=os.path.join(root,sub[1:].replace('/',os.sep)) 73info=os.lstat(full) 74 75m=info.st_mode 76ifstat.S_ISDIR(m): 77ifsub!='/': 78yield"D %s %s"%(int(info.st_mtime),sub) 79items=os.listdir(full) 80items.sort() 81subdir=sub 82ifnotsubdir.endswith('/'): 83subdir+='/' 84forxinitems: 85foryinrecurse(subdir+x): 86yieldy 87return 88 89assertsub[1:] 90leaf=os.path.basename(sub[1:]) 91ifstat.S_ISREG(m): 92withopen(full,'rb')asstream: 93d=sha1_new(stream.read()).hexdigest()# XXX could be very large! 94ifm&0o111: 95yield"X %s %s %s %s"%(d,int(info.st_mtime),info.st_size,leaf) 96else: 97yield"F %s %s %s %s"%(d,int(info.st_mtime),info.st_size,leaf) 98elifstat.S_ISLNK(m): 99target=os.readlink(full).encode('utf-8')100d=sha1_new(target).hexdigest()101# Note: Can't use utime on symlinks, so skip mtime102# Note: eCryptfs may report length as zero, so count ourselves instead103yield"S %s %s %s"%(d,len(target),leaf)104else:105raiseSafeException(_("Unknown object '%s' (not a file, directory or symlink)")%106full)
117"""Look-up an L{Algorithm} by name.118 @type name: str119 @rtype: L{Algorithm}120 @raise BadDigest: if the name is unknown."""121try:122returnalgorithms[name]123exceptKeyError:124raiseBadDigest(_("Unknown algorithm '%s'")%name)
133"""Writes a .manifest file into 'dir', and returns the digest.134 You should call fixup_permissions before this to ensure that the permissions are correct.135 On exit, dir itself has mode 555. Subdirectories are not changed.136 @param dir: root of the implementation137 @type dir: str138 @param digest_or_alg: should be an instance of Algorithm. Passing a digest here is deprecated.139 @type digest_or_alg: L{Algorithm}"""140mfile=os.path.join(dir,'.manifest')141ifos.path.islink(mfile)oros.path.exists(mfile):142raiseSafeException(_("Directory '%s' already contains a .manifest file!")%dir)143manifest=''144ifisinstance(digest_or_alg,Algorithm):145alg=digest_or_alg146digest=alg.new_digest()147else:148digest=digest_or_alg149alg=get_algorithm('sha1')150forlineinalg.generate_manifest(dir):151manifest+=line+'\n'152manifest=manifest.encode('utf-8')153digest.update(manifest)154155os.chmod(dir,0o755)156withopen(mfile,'wb')asstream:157os.chmod(dir,0o555)158stream.write(manifest)159os.chmod(mfile,0o444)160returndigest
163"""Take an ID in the form 'alg=value' and return a tuple (alg, value).164 @type id: str165 @rtype: (L{Algorithm}, str)166 @raise BadDigest: if the algorithm isn't known or the ID has the wrong format."""167alg,digest=parse_algorithm_digest_pair(id)168return(get_algorithm(alg),digest)
171"""Copy path src to dest, checking that the contents give the right digest.172 dest must not exist. New file is created with a mode of 'mode & umask'.173 @param src: source filename174 @type src: str175 @param dest: target filename176 @type dest: str177 @param mode: target mode178 @type mode: int179 @param alg: algorithm to generate digest180 @type alg: L{Algorithm}181 @param required_digest: expected digest value182 @type required_digest: str183 @raise BadDigest: the contents of the file don't match required_digest"""184withopen(src,'rb')assrc_obj:185dest_fd=os.open(dest,os.O_WRONLY|os.O_CREAT|os.O_EXCL,mode)186try:187digest=alg.new_digest()188whileTrue:189data=src_obj.read(256)190ifnotdata:break191digest.update(data)192whiledata:193written=os.write(dest_fd,data)194assertwritten>=0195data=data[written:]196finally:197os.close(dest_fd)198actual=digest.hexdigest()199ifactual==required_digest:return200os.unlink(dest)201raiseBadDigest(_("Copy failed: file '%(src)s' has wrong digest (may have been tampered with)\n"202"Expected: %(required_digest)s\n"203"Actual: %(actual_digest)s")%{'src':src,'required_digest':required_digest,'actual_digest':actual})
206"""Ensure that directory 'dir' generates the given digest.207 For a non-error return:208 - Dir's name must be a digest (in the form "alg=value")209 - The calculated digest of the contents must match this name.210 - If there is a .manifest file, then its digest must also match.211 @type root: str212 @type required_digest: str | None213 @raise BadDigest: if verification fails."""214ifrequired_digestisNone:215required_digest=os.path.basename(root)216alg=splitID(required_digest)[0]217218digest=alg.new_digest()219lines=[]220forlineinalg.generate_manifest(root):221line+='\n'222digest.update(line.encode('utf-8'))223lines.append(line)224actual_digest=alg.getID(digest)225226manifest_file=os.path.join(root,'.manifest')227ifos.path.isfile(manifest_file):228digest=alg.new_digest()229withopen(manifest_file,'rb')asstream:230digest.update(stream.read())231manifest_digest=alg.getID(digest)232else:233manifest_digest=None234235ifrequired_digest==actual_digest==manifest_digest:236return237238error=BadDigest(_("Cached item does NOT verify."))239240error.detail=_(" Expected: %(required_digest)s\n"241" Actual: %(actual_digest)s\n"242".manifest digest: %(manifest_digest)s\n\n") \ 243%{'required_digest':required_digest,'actual_digest':actual_digest,'manifest_digest':manifest_digestor_('No .manifest file')}244245ifmanifest_digestisNone:246error.detail+=_("No .manifest, so no further details available.")247elifmanifest_digest==actual_digest:248error.detail+=_("The .manifest file matches the actual contents. Very strange!")249elifmanifest_digest==required_digest:250importdifflib251withopen(manifest_file,'rt')asstream:252diff=difflib.unified_diff(stream.readlines(),lines,253'Recorded','Actual')254error.detail+=_("The .manifest file matches the directory name.\n" \ 255"The contents of the directory have changed:\n")+ \ 256''.join(diff)257elifrequired_digest==actual_digest:258error.detail+=_("The directory contents are correct, but the .manifest file is wrong!")259else:260error.detail+=_("The .manifest file matches neither of the other digests. Odd.")261raiseerror
262263# XXX: Be more careful about the source tree changing under us. In particular, what happens if:264# - A regualar file suddenly turns into a symlink?265# - We find a device file (users can hard-link them if on the same device)
267"""Copy directory source to be a subdirectory of target if it matches the required_digest.268 manifest_data is normally source/.manifest. source and manifest_data are not trusted269 (will typically be under the control of another user).270 The copy is first done to a temporary directory in target, then renamed to the final name271 only if correct. Therefore, an invalid 'target/required_digest' will never exist.272 A successful return means than target/required_digest now exists (whether we created it or not).273 @type source: str274 @type target: str275 @type manifest_data: str276 @type required_digest: str"""277importtempfile278279alg,digest_value=splitID(required_digest)280281ifisinstance(alg,OldSHA1):282raiseSafeException(_("Sorry, the 'sha1' algorithm does not support copying."))283284digest=alg.new_digest()285digest.update(manifest_data)286manifest_digest=alg.getID(digest)287288ifmanifest_digest!=required_digest:289raiseBadDigest(_("Manifest has been tampered with!\n"290"Manifest digest: %(actual_digest)s\n"291"Directory name : %(required_digest)s")292%{'actual_digest':manifest_digest,'required_digest':required_digest})293294target_impl=os.path.join(target,required_digest)295ifos.path.isdir(target_impl):296logger.info(_("Target directory '%s' already exists"),target_impl)297return298299# We've checked that the source's manifest matches required_digest, so it300# is what we want. Make a list of all the files we need to copy...301302wanted=_parse_manifest(manifest_data.decode('utf-8'))303304tmpdir=tempfile.mkdtemp(prefix='tmp-copy-',dir=target)305try:306_copy_files(alg,wanted,source,tmpdir)307308ifwanted:309raiseSafeException(_('Copy failed; files missing from source:')+'\n- '+310'\n- '.join(wanted.keys()))311312# Make directories read-only (files are already RO)313forroot,dirs,filesinos.walk(tmpdir):314fordindirs:315path=os.path.join(root,d)316mode=os.stat(path).st_mode317os.chmod(path,mode&0o555)318319# Check that the copy is correct320actual_digest=alg.getID(add_manifest_file(tmpdir,alg))321ifactual_digest!=required_digest:322raiseSafeException(_("Copy failed; double-check of target gave the wrong digest.\n"323"Unless the target was modified during the copy, this is a BUG\n"324"in 0store and should be reported.\n"325"Expected: %(required_digest)s\n"326"Actual: %(actual_digest)s")%{'required_digest':required_digest,'actual_digest':actual_digest})327try:328os.chmod(tmpdir,0o755)# need write permission to rename on MacOS X329os.rename(tmpdir,target_impl)330os.chmod(target_impl,0o555)331tmpdir=None332exceptOSError:333ifnotos.path.isdir(target_impl):334raise335# else someone else installed it already - return success336finally:337iftmpdirisnotNone:338logger.info(_("Deleting tmpdir '%s'")%tmpdir)339fromzeroinstall.supportimportro_rmtree340ro_rmtree(tmpdir)
343"""Parse a manifest file.344 @param manifest_data: the contents of the manifest file345 @type manifest_data: str346 @return: a mapping from paths to information about that path347 @rtype: {str: tuple}"""348wanted={}349dir=''350forlineinmanifest_data.split('\n'):351ifnotline:break352ifline[0]=='D':353data=line.split(' ',1)354iflen(data)!=2:raiseBadDigest(_("Bad line '%s'")%line)355path=data[-1]356ifnotpath.startswith('/'):raiseBadDigest(_("Not absolute: '%s'")%line)357path=path[1:]358dir=path359elifline[0]=='S':360data=line.split(' ',3)361path=os.path.join(dir,data[-1])362iflen(data)!=4:raiseBadDigest(_("Bad line '%s'")%line)363else:364data=line.split(' ',4)365path=os.path.join(dir,data[-1])366iflen(data)!=5:raiseBadDigest(_("Bad line '%s'")%line)367ifpathinwanted:368raiseBadDigest(_('Duplicate entry "%s"')%line)369wanted[path]=data[:-1]370returnwanted
373"""Scan for files under 'source'. For each one:374 If it is in wanted and has the right details (or they can be fixed; e.g. mtime),375 then copy it into 'target'.376 If it's not in wanted, warn and skip it.377 On exit, wanted contains only files that were not found.378 @type alg: L{Algorithm}379 @type wanted: {str: tuple}380 @type source: str381 @type target: str"""382dir=''383forlineinalg.generate_manifest(source):384ifline[0]=='D':385type,name=line.split(' ',1)386assertname.startswith('/')387dir=name[1:]388path=dir389elifline[0]=='S':390type,actual_digest,actual_size,name=line.split(' ',3)391path=os.path.join(dir,name)392else:393assertline[0]in'XF'394type,actual_digest,actual_mtime,actual_size,name=line.split(' ',4)395path=os.path.join(dir,name)396try:397required_details=wanted.pop(path)398exceptKeyError:399logger.warning(_("Skipping file not in manifest: '%s'"),path)400continue401ifrequired_details[0]!=type:402raiseBadDigest(_("Item '%s' has wrong type!")%path)403iftype=='D':404os.mkdir(os.path.join(target,path))405eliftypein'XF':406required_type,required_digest,required_mtime,required_size=required_details407ifrequired_size!=actual_size:408raiseSafeException(_("File '%(path)s' has wrong size (%(actual_size)s bytes, but should be "409"%(required_size)s according to manifest)")%410{'path':path,'actual_size':actual_size,'required_size':required_size})411required_mtime=int(required_mtime)412dest_path=os.path.join(target,path)413iftype=='X':414mode=0o555415else:416mode=0o444417copy_with_verify(os.path.join(source,path),418dest_path,419mode,420alg,421required_digest)422os.utime(dest_path,(required_mtime,required_mtime))423eliftype=='S':424required_type,required_digest,required_size=required_details425ifrequired_size!=actual_size:426raiseSafeException(_("Symlink '%(path)s' has wrong size (%(actual_size)s bytes, but should be "427"%(required_size)s according to manifest)")%428{'path':path,'actual_size':actual_size,'required_size':required_size})429symlink_target=os.readlink(os.path.join(source,path))430symlink_digest=alg.new_digest()431symlink_digest.update(symlink_target.encode('utf-8'))432ifsymlink_digest.hexdigest()!=required_digest:433raiseSafeException(_("Symlink '%(path)s' has wrong target (digest should be "434"%(digest)s according to manifest)")%{'path':path,'digest':required_digest})435dest_path=os.path.join(target,path)436os.symlink(symlink_target,dest_path)437else:438raiseSafeException(_("Unknown manifest type %(type)s for '%(path)s'")%{'type':type,'path':path})
452"""@type root: str"""453defrecurse(sub):454# To ensure that a line-by-line comparison of the manifests455# is possible, we require that filenames don't contain newlines.456# Otherwise, you can name a file so that the part after the \n457# would be interpreted as another line in the manifest.458if'\n'insub:raiseBadDigest(_("Newline in filename '%s'")%sub)459assertsub.startswith('/')460461full=os.path.join(root,sub[1:])462info=os.lstat(full)463new_digest=self.new_digest464465m=info.st_mode466ifnotstat.S_ISDIR(m):raiseException(_('Not a directory: "%s"')%full)467ifsub!='/':468yield"D %s"%sub469items=os.listdir(full)470items.sort()471dirs=[]472forleafinitems:473path=os.path.join(root,sub[1:],leaf)474info=os.lstat(path)475m=info.st_mode476477ifstat.S_ISREG(m):478ifleaf=='.manifest':continue479480withopen(path,'rb')asstream:481d=new_digest(stream.read()).hexdigest()482ifm&0o111:483yield"X %s %s %s %s"%(d,int(info.st_mtime),info.st_size,leaf)484else:485yield"F %s %s %s %s"%(d,int(info.st_mtime),info.st_size,leaf)486elifstat.S_ISLNK(m):487target=os.readlink(path).encode('utf-8')488d=new_digest(target).hexdigest()489# Note: Can't use utime on symlinks, so skip mtime490# Note: eCryptfs may report length as zero, so count ourselves instead491yield"S %s %s %s"%(d,len(target),leaf)492elifstat.S_ISDIR(m):493dirs.append(leaf)494else:495raiseSafeException(_("Unknown object '%s' (not a file, directory or symlink)")%496path)497498ifnotsub.endswith('/'):499sub+='/'500forxindirs:501# Note: "sub" is always Unix style. Don't use os.path.join here.502foryinrecurse(sub+x):yieldy503return
508"""@rtype: str"""509ifself.namein('sha1new','sha256'):510digest_str=digest.hexdigest()511else:512# Base32-encode newer algorithms to make the digest shorter.513# We can't use base64 as Windows is case insensitive.514# There's no need for padding (and = characters in paths cause problems for some software).515digest_str=base64.b32encode(digest.digest()).rstrip(b'=').decode('ascii')516returnformat_algorithm_digest_pair(self.name,digest_str)
527"""Set permissions recursively for children of root:528 - If any X bit is set, they all must be.529 - World readable, non-writable.530 @type root: str531 @raise Exception: if there are unsafe special bits set (setuid, etc)."""532533formain,dirs,filesinos.walk(root):534forxin['.']+files:535full=os.path.join(main,x)536537raw_mode=os.lstat(full).st_mode538ifstat.S_ISLNK(raw_mode):continue539540mode=stat.S_IMODE(raw_mode)541ifmode&~0o777:542raiseException(_("Unsafe mode: extracted file '%(filename)s' had special bits set in mode '%(mode)s'")%{'filename':full,'mode':oct(mode)})543ifmode&0o111:544os.chmod(full,0o555)545else:546os.chmod(full,0o444)