[yocto] [PATCH][autobuilder] lib/buildsteps.py: Add BitbakeShellCommand class for dump bitbake logs

[Flanagan, Elizabeth]

On 1 March 2016 at 12:54, Flanagan, Elizabeth
<elizabeth.flanagan at intel.com> wrote:
> On 1 March 2016 at 00:36, Aníbal Limón <anibal.limon at linux.intel.com> wrote:
>> The BitbakeShellCommand is a new class for store common methods when
>> is aim to run bitbake inside an step.
>>
>> This commit overrides commandComplete method for search in bitbake
>> stdout if ERROR exists and if found then open the bitbake log and
>> dumps into new bitbake_logs output.

So, obviously some issues here with using open (controller doesn't
have access to the worker filesystem).

A thought. Scroll the logs, setting a property with the failures and
log location and use that at the end to have a
PublishBuildbotErrorLogs to DEST?

-b

>
> Yes, I like this a lot. I'm going to pull it into master-next and test
> it for a bit. Expect it in master/production in a few days.
>
> -b
>
>>
>> Signed-off-by: Aníbal Limón <anibal.limon at linux.intel.com>
>> ---
>>  .../autobuilder/buildsteps/BuildImages.py          |  4 +++-
>>  .../autobuilder/buildsteps/BuildToolchainImages.py |  4 +++-
>>  .../autobuilder/buildsteps/GetBitbakeVersion.py    |  4 +++-
>>  .../autobuilder/buildsteps/RunESDKSanityTests.py   |  4 +++-
>>  .../autobuilder/buildsteps/RunSDKSanityTests.py    |  4 +++-
>>  .../autobuilder/buildsteps/RunSanityTests.py       |  4 +++-
>>  .../site-packages/autobuilder/lib/buildsteps.py    | 28 ++++++++++++++++++++++
>>  7 files changed, 46 insertions(+), 6 deletions(-)
>>
>> diff --git a/lib/python2.7/site-packages/autobuilder/buildsteps/BuildImages.py b/lib/python2.7/site-packages/autobuilder/buildsteps/BuildImages.py
>> index 4987ef3..5ed6e1a 100644
>> --- a/lib/python2.7/site-packages/autobuilder/buildsteps/BuildImages.py
>> +++ b/lib/python2.7/site-packages/autobuilder/buildsteps/BuildImages.py
>> @@ -17,7 +17,9 @@ from distutils.version import StrictVersion
>>  from buildbot.status.results import SUCCESS
>>  import os
>>
>> -class BuildImages(ShellCommand):
>> +from lib.buildsteps import BitbakeShellCommand
>> +
>> +class BuildImages(BitbakeShellCommand):
>>      haltOnFailure = False
>>      flunkOnFailure = True
>>      name = "BuildImages"
>> diff --git a/lib/python2.7/site-packages/autobuilder/buildsteps/BuildToolchainImages.py b/lib/python2.7/site-packages/autobuilder/buildsteps/BuildToolchainImages.py
>> index 51069c2..0594ac2 100644
>> --- a/lib/python2.7/site-packages/autobuilder/buildsteps/BuildToolchainImages.py
>> +++ b/lib/python2.7/site-packages/autobuilder/buildsteps/BuildToolchainImages.py
>> @@ -15,7 +15,9 @@ from buildbot.steps.shell import ShellCommand
>>  from buildbot.process.buildstep import LogLineObserver
>>  import os
>>
>> -class BuildToolchainImages(ShellCommand):
>> +from lib.buildsteps import BitbakeShellCommand
>> +
>> +class BuildToolchainImages(BitbakeShellCommand):
>>      haltOnFailure = False
>>      flunkOnFailure = True
>>      name = "Building Toolchain Images"
>> diff --git a/lib/python2.7/site-packages/autobuilder/buildsteps/GetBitbakeVersion.py b/lib/python2.7/site-packages/autobuilder/buildsteps/GetBitbakeVersion.py
>> index dc6c25b..94b5efe 100644
>> --- a/lib/python2.7/site-packages/autobuilder/buildsteps/GetBitbakeVersion.py
>> +++ b/lib/python2.7/site-packages/autobuilder/buildsteps/GetBitbakeVersion.py
>> @@ -16,7 +16,9 @@ from buildbot.process.properties import WithProperties
>>  from twisted.python import log
>>  from autobuilder.config import *
>>
>> -class GetBitbakeVersion(ShellCommand):
>> +from lib.buildsteps import BitbakeShellCommand
>> +
>> +class GetBitbakeVersion(BitbakeShellCommand):
>>      haltOnFailure = False
>>      flunkOnFailure = False
>>      name = "GetBitbakeVersion"
>> diff --git a/lib/python2.7/site-packages/autobuilder/buildsteps/RunESDKSanityTests.py b/lib/python2.7/site-packages/autobuilder/buildsteps/RunESDKSanityTests.py
>> index e3df6c8..f2817d4 100644
>> --- a/lib/python2.7/site-packages/autobuilder/buildsteps/RunESDKSanityTests.py
>> +++ b/lib/python2.7/site-packages/autobuilder/buildsteps/RunESDKSanityTests.py
>> @@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
>>  from twisted.python import log as tlog
>>  import os, re
>>
>> -class RunESDKSanityTests(ShellCommand):
>> +from lib.buildsteps import BitbakeShellCommand
>> +
>> +class RunESDKSanityTests(BitbakeShellCommand):
>>
>>      haltOnFailure = False
>>      flunkOnFailure = True
>> diff --git a/lib/python2.7/site-packages/autobuilder/buildsteps/RunSDKSanityTests.py b/lib/python2.7/site-packages/autobuilder/buildsteps/RunSDKSanityTests.py
>> index cc0321f..1eeb4d3 100644
>> --- a/lib/python2.7/site-packages/autobuilder/buildsteps/RunSDKSanityTests.py
>> +++ b/lib/python2.7/site-packages/autobuilder/buildsteps/RunSDKSanityTests.py
>> @@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
>>  from twisted.python import log as tlog
>>  import os, re
>>
>> -class RunSDKSanityTests(ShellCommand):
>> +from lib.buildsteps import BitbakeShellCommand
>> +
>> +class RunSDKSanityTests(BitbakeShellCommand):
>>
>>      haltOnFailure = False
>>      flunkOnFailure = True
>> diff --git a/lib/python2.7/site-packages/autobuilder/buildsteps/RunSanityTests.py b/lib/python2.7/site-packages/autobuilder/buildsteps/RunSanityTests.py
>> index 62f04af..15a714c 100644
>> --- a/lib/python2.7/site-packages/autobuilder/buildsteps/RunSanityTests.py
>> +++ b/lib/python2.7/site-packages/autobuilder/buildsteps/RunSanityTests.py
>> @@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
>>  from twisted.python import log as tlog
>>  import os, re
>>
>> -class RunSanityTests(ShellCommand):
>> +from lib.buildsteps import BitbakeShellCommand
>> +
>> +class RunSanityTests(BitbakeShellCommand):
>>
>>      haltOnFailure = False
>>      flunkOnFailure = True
>> diff --git a/lib/python2.7/site-packages/autobuilder/lib/buildsteps.py b/lib/python2.7/site-packages/autobuilder/lib/buildsteps.py
>> index 3693a7a..4d2787d 100644
>> --- a/lib/python2.7/site-packages/autobuilder/lib/buildsteps.py
>> +++ b/lib/python2.7/site-packages/autobuilder/lib/buildsteps.py
>> @@ -11,6 +11,8 @@ __email__ = "anibal.limon at linux.intel.com"
>>  '''
>>
>>  import os
>> +import re
>> +
>>  from buildbot.steps.shell import ShellCommand
>>
>>  DEFAULT_SHELL = 'bash'
>> @@ -40,3 +42,29 @@ class ShellCommandCleanEnv(ShellCommand):
>>              pe_cmd += "%s=\"$%s\" " % (pe, pe)
>>
>>          return "env -i %s %s -c " % (pe_cmd, shell)
>> +
>> +class BitbakeShellCommand(ShellCommand):
>> +    def commandComplete(self, cmd):
>> +        if cmd.didFail():
>> +            log = self.addLog('bitbake_logs')
>> +            paths_displayed = []
>> +
>> +            rexp = re.compile("^ERROR: Logfile of failure stored in: (.*)$")
>> +            out = cmd.logs['stdio'].getText()
>> +            for line in out.split('\n'):
>> +                m = rexp.match(line)
>> +                if m:
>> +                    path = m.group(1)
>> +
>> +                    if path in paths_displayed:
>> +                        continue
>> +
>> +                    paths_displayed.append(path)
>> +
>> +                    log.addStdout("Output of: %s\n" % path)
>> +                    with open(path, "r") as f:
>> +                        log.addStdout(f.read())
>> +                    log.addStdout("End output of: %s\n" % path)
>> +                    log.addStdout("\n\n\n\n")
>> +            log.finish()
>> +
>> --
>> 2.1.4
>>
>
>
>
> --
> Elizabeth Flanagan
> Yocto Project
> Build and Release



-- 
Elizabeth Flanagan
Yocto Project
Build and Release