[yocto] [[yocto-autobuilder][PATCHv2] 05/15] buildbot: Add support for DirectoryDownload transfer step
Aníbal Limón
anibal.limon at linux.intel.com
Tue Jun 21 16:07:42 PDT 2016
The new DirectoryDownload transfer step provides support for
download a directory from master to slave in specific location.
The current implementation extends FileDownload step but it creates
a tar file (may be compressed) and download it from master to slave,
finally slave decompress the tarfile into specific directory.
Signed-off-by: Aníbal Limón <anibal.limon at linux.intel.com>
---
.../buildbot/steps/transfer.py | 86 +++++++++++++++++++++-
.../buildslave/commands/registry.py | 1 +
.../buildslave/commands/transfer.py | 79 ++++++++++++++++++++
3 files changed, 164 insertions(+), 2 deletions(-)
diff --git a/lib/python2.7/site-packages/buildbot-0.8.8-py2.7.egg/buildbot/steps/transfer.py b/lib/python2.7/site-packages/buildbot-0.8.8-py2.7.egg/buildbot/steps/transfer.py
index 09b3750..fb49597 100644
--- a/lib/python2.7/site-packages/buildbot-0.8.8-py2.7.egg/buildbot/steps/transfer.py
+++ b/lib/python2.7/site-packages/buildbot-0.8.8-py2.7.egg/buildbot/steps/transfer.py
@@ -170,7 +170,6 @@ class _DirectoryWriter(_FileWriter):
archive.close()
os.remove(self.tarname)
-
def makeStatusRemoteCommand(step, remote_command, args):
self = buildstep.RemoteCommand(remote_command, args, decodeRC={None:SUCCESS, 0:SUCCESS})
callback = lambda arg: step.step_status.addLog('stdio')
@@ -365,7 +364,6 @@ class DirectoryUpload(_TransferBuildStep):
return BuildStep.finished(self, FAILURE)
return BuildStep.finished(self, SUCCESS)
-
class _FileReader(pb.Referenceable):
"""
Helper class that acts as a file-object with read access
@@ -463,6 +461,90 @@ class FileDownload(_TransferBuildStep):
d = self.runCommand(self.cmd)
d.addCallback(self.finished).addErrback(self.failed)
+class DirectoryDownload(_TransferBuildStep):
+
+ name = 'download'
+
+ renderables = [ 'mastersrc', 'slavedest' ]
+
+ def __init__(self, mastersrc, slavedest,
+ workdir=None, maxsize=None, blocksize=16*1024,
+ compress=None, mode=None, **buildstep_kwargs):
+ BuildStep.__init__(self, **buildstep_kwargs)
+
+ self.mastersrc = mastersrc
+ self.slavedest = slavedest
+ self.workdir = workdir
+ self.maxsize = maxsize
+ self.blocksize = blocksize
+ if compress not in (None, 'gz', 'bz2'):
+ config.error(
+ "'compress' must be one of None, 'gz', or 'bz2'")
+ self.compress = compress
+ self.mode = mode
+
+ def start(self):
+ version = self.slaveVersion("downloadDirectory")
+
+ if not version:
+ m = "slave is too old, does not know about uploadDirectory"
+ raise BuildSlaveTooOldError(m)
+
+ # we rely upon the fact that the buildmaster runs chdir'ed into its
+ # basedir to make sure that relative paths in mastersrc are expanded
+ # properly. TODO: maybe pass the master's basedir all the way down
+ # into the BuildStep so we can do this better.
+ source = os.path.expanduser(self.mastersrc)
+
+ slavedest = self.slavedest
+
+ log.msg("DirectoryDownload started, from master %r to slave %r"
+ % (source, slavedest))
+
+ self.step_status.setText(['downloading', "to", slavedest])
+
+ # setup structures for reading the file
+ try:
+ fd, self.tarname = tempfile.mkstemp()
+ fileobj = os.fdopen(fd, 'w')
+ if self.compress == 'bz2':
+ mode='w|bz2'
+ elif self.compress == 'gz':
+ mode='w|gz'
+ else:
+ mode = 'w'
+ archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj)
+ archive.add(source, '')
+ archive.close()
+ fileobj.close()
+
+ fp = open(self.tarname, 'rb')
+ except IOError:
+ # if file does not exist, bail out with an error
+ self.addCompleteLog('stderr',
+ 'Directory %r not available at master' % source)
+ # TODO: once BuildStep.start() gets rewritten to use
+ # maybeDeferred, just re-raise the exception here.
+ eventually(BuildStep.finished, self, FAILURE)
+ return
+
+ fileReader = _FileReader(fp)
+
+ # default arguments
+ args = {
+ 'slavedest': slavedest,
+ 'maxsize': self.maxsize,
+ 'reader': fileReader,
+ 'blocksize': self.blocksize,
+ 'workdir': self._getWorkdir(),
+ 'mode': self.mode,
+ 'compress': self.compress,
+ }
+
+ self.cmd = makeStatusRemoteCommand(self, 'downloadDirectory', args)
+ d = self.runCommand(self.cmd)
+ d.addCallback(self.finished).addErrback(self.failed)
+
class StringDownload(_TransferBuildStep):
name = 'string_download'
diff --git a/lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/registry.py b/lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/registry.py
index 080e814..22dc3ee 100644
--- a/lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/registry.py
+++ b/lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/registry.py
@@ -21,6 +21,7 @@ commandRegistry = {
"uploadFile" : "buildslave.commands.transfer.SlaveFileUploadCommand",
"uploadDirectory" : "buildslave.commands.transfer.SlaveDirectoryUploadCommand",
"downloadFile" : "buildslave.commands.transfer.SlaveFileDownloadCommand",
+ "downloadDirectory" : "buildslave.commands.transfer.SlaveDirectoryDownloadCommand",
"svn" : "buildslave.commands.svn.SVN",
"bk" : "buildslave.commands.bk.BK",
"cvs" : "buildslave.commands.cvs.CVS",
diff --git a/lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/transfer.py b/lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/transfer.py
index 670c54e..e51e8b9 100644
--- a/lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/transfer.py
+++ b/lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/transfer.py
@@ -354,3 +354,82 @@ class SlaveFileDownloadCommand(TransferCommand):
self.fp.close()
return TransferCommand.finished(self, res)
+
+class SlaveDirectoryDownloadCommand(SlaveFileDownloadCommand):
+ debug = False
+
+ def setup(self, args):
+ self.workdir = args['workdir']
+
+ self.dirname = os.path.expanduser(args['slavedest'])
+ self.reader = args['reader']
+ self.bytes_remaining = args['maxsize']
+ self.blocksize = args['blocksize']
+ self.compress = args['compress']
+ self.mode = args['mode']
+ self.stderr = None
+ self.rc = 0
+
+ def start(self):
+ if self.debug:
+ log.msg('SlaveDirectoryDownloadCommand starting')
+
+ if not os.path.exists(self.dirname):
+ os.makedirs(self.dirname)
+
+ try:
+ fd, self.tarname = tempfile.mkstemp()
+ self.fp = os.fdopen(fd, 'w')
+ if self.debug:
+ log.msg("Opened '%s' for download" % self.tarname)
+ except IOError:
+ # TODO: this still needs cleanup
+ self.fp = None
+ self.stderr = "Cannot open file '%s' for download" % self.tarname
+ self.rc = 1
+ if self.debug:
+ log.msg("Cannot open file '%s' for download" % self.tarname)
+
+ d = defer.Deferred()
+ self._reactor.callLater(0, self._loop, d)
+ def _close(res):
+ # close the file, but pass through any errors from _loop
+ d1 = self.reader.callRemote('close')
+ d1.addErrback(log.err, 'while trying to close reader')
+ d1.addCallback(lambda ignored: res)
+ return d1
+ d.addBoth(_close)
+ d.addBoth(self.finished)
+ return d
+
+ def finished(self, res):
+ if self.fp is not None:
+ self.fp.close()
+
+ # decompress
+ if self.compress == 'bz2':
+ mode='r|bz2'
+ elif self.compress == 'gz':
+ mode='r|gz'
+ else:
+ mode = 'r'
+ if not hasattr(tarfile.TarFile, 'extractall'):
+ tarfile.TarFile.extractall = _extractall
+ archive = tarfile.open(name=self.tarname, mode=mode)
+ archive.extractall(path=self.dirname)
+ archive.close()
+ os.remove(self.tarname)
+
+ # note: there is a brief window during which the new file
+ # will have the buildslave's default (umask) mode before we
+ # set the new one. Don't use this mode= feature to keep files
+ # private: use the buildslave's umask for that instead. (it
+ # is possible to call os.umask() before and after the open()
+ # call, but cleaning up from exceptions properly is more of a
+ # nuisance that way).
+ if self.mode is not None:
+ for f in os.listdir(self.dirname):
+ full_f = os.path.join(self.dirname, f)
+ os.chmod(full_f, self.mode)
+
+ return TransferCommand.finished(self, res)
--
2.1.4
More information about the yocto
mailing list