[yocto] [PATCH 10/10][auh] upgradehelper: Add support for generate buildhistory recipe diff's
Aníbal Limón
anibal.limon at linux.intel.com
Wed Jul 29 13:50:53 PDT 2015
Now AUH generates buildhistory diff when recipe was upgraded
successfully.
For enable this feature set into local.conf.
INHERIT += "buildhistory"
BUILDHISTORY_COMMIT = "1"
Summary of the changes,
- bitbake.py: Enable environment generation without recipe,
removes Buildhistory class.
- buildhistory.py: Add buildhistory class that generates initial
buildhistory revision and keeps track of build revisions for
generate diff when build is successful.
- upgradehelper.py: Add buildhistory steps for upgrade, add
support for detect when buildhistory is enabled loading the
environment without recipe.
[YOCTO #7175]
Signed-off-by: Aníbal Limón <anibal.limon at linux.intel.com>
---
bitbake.py | 41 ++------------------
buildhistory.py | 75 ++++++++++++++++++++++++++++++++++++
errors.py | 6 +++
upgradehelper.py | 115 ++++++++++++++++++++++++++++++++++++++++---------------
4 files changed, 170 insertions(+), 67 deletions(-)
create mode 100644 buildhistory.py
diff --git a/bitbake.py b/bitbake.py
index e23dc28..a1587ce 100644
--- a/bitbake.py
+++ b/bitbake.py
@@ -44,7 +44,7 @@ class Bitbake(object):
self.log_dir = None
super(Bitbake, self).__init__()
- def _cmd(self, recipe, options=None, env_var=None, output_filter=None):
+ def _cmd(self, recipe=None, options=None, env_var=None, output_filter=None):
cmd = ""
if env_var is not None:
cmd += env_var + " "
@@ -52,7 +52,8 @@ class Bitbake(object):
if options is not None:
cmd += options + " "
- cmd += recipe
+ if recipe is not None:
+ cmd += recipe
if output_filter is not None:
cmd += ' | grep ' + output_filter
@@ -78,7 +79,7 @@ class Bitbake(object):
def get_stdout_log(self):
return os.path.join(self.log_dir, BITBAKE_ERROR_LOG)
- def env(self, recipe):
+ def env(self, recipe=None):
return self._cmd(recipe, "-e", output_filter="-v \"^#\"")
def fetch(self, recipe):
@@ -104,37 +105,3 @@ class Bitbake(object):
def dependency_graph(self, package_list):
return self._cmd(package_list, "-g")
-
-class BuildHistory(object):
- def __init__(self, build_dir):
- self.build_dir = build_dir
- self.work_dir = None
-
- def set_work_dir(self, work_dir):
- self.work_dir = work_dir
-
- # Return True if buildhistory-diff gives output
- def diff(self, revision_steps):
- os.chdir(self.build_dir)
- cmd = "buildhistory-diff HEAD~" + str(revision_steps)
-
- try:
- stdout, stderr = bb.process.run(cmd)
- # Write diff output to log file if there is any
-
- if stdout and os.path.exists(self.work_dir):
- with open(os.path.join(self.work_dir, "buildhistory.txt"), "w+") as log:
- log.write(stdout)
- return True
- except bb.process.ExecutionError as e:
- for line in e.stdout.split('\n'):
- if line.find("Buildhistory directory \"buildhistory/\" does not exist") == 0:
- C(" \"buildhistory.bbclass\" not inherited. Consider adding "
- "the following to your local.conf:\n\n"
- "INHERIT =+ \"buildhistory\"\n"
- "BUILDHISTORY_COMMIT = \"1\"\n\n"
- "Do not remove any other inherited class in the process (e.g. distrodata)\n")
- exit(1)
-
- return False
-
diff --git a/buildhistory.py b/buildhistory.py
new file mode 100644
index 0000000..1732f23
--- /dev/null
+++ b/buildhistory.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# vim: set ts=4 sw=4 et:
+#
+# Copyright (c) 2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+
+import os
+import logging as log
+from logging import info as I
+from logging import debug as D
+from logging import error as E
+from logging import critical as C
+import sys
+from errors import *
+
+from bitbake import *
+from git import Git
+
+os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \
+ " BUILDHISTORY_DIR"
+
+class BuildHistory(object):
+ def __init__(self, bb, pn, workdir):
+ self.bb = bb
+ self.pn = pn
+ self.workdir = workdir
+ self.revs = []
+
+ self.buildhistory_dir = os.path.join(self.workdir, 'buildhistory')
+ if not os.path.exists(self.buildhistory_dir):
+ os.mkdir(self.buildhistory_dir)
+
+ self.git = Git(self.buildhistory_dir)
+
+ os.environ["BUILDHISTORY_DIR"] = self.buildhistory_dir
+
+ def init(self, machines):
+ self.bb.cleanall(self.pn)
+ for machine in machines:
+ self.bb.complete(self.pn, machine)
+ self.revs.append(self.git.last_commit("master"))
+
+ def add(self):
+ self.revs.append(self.git.last_commit("master"))
+
+ def diff(self):
+ rev_initial = self.revs[0]
+ rev_final = self.revs[-1]
+
+ cmd = "buildhistory-diff -a -p %s %s %s" % (self.buildhistory_dir,
+ rev_initial, rev_final)
+
+ try:
+ stdout, stderr = bb.process.run(cmd)
+
+ if stdout and os.path.exists(self.workdir):
+ with open(os.path.join(self.workdir, "buildhistory-diff.txt"),
+ "w+") as log:
+ log.write(stdout)
+ except bb.process.ExecutionError as e:
+ W( "%s: Buildhistory checking fails\n%s" % (self.pn, e.stdout))
diff --git a/errors.py b/errors.py
index 7194944..1504fa5 100644
--- a/errors.py
+++ b/errors.py
@@ -85,3 +85,9 @@ class UpgradeNotNeededError(Error):
def __str__(self):
return "Failed(up to date)"
+class EmptyEnvError(Error):
+ def __init__(self, stdout):
+ super(EmptyEnvError, self).__init__("Empty environment returned", stdout)
+
+ def __str__(self):
+ return "Failed(get_env)"
diff --git a/upgradehelper.py b/upgradehelper.py
index 4d8685c..a8bc5ec 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -44,6 +44,7 @@ import shutil
from errors import *
from git import Git
from bitbake import Bitbake
+from buildhistory import BuildHistory
from emailhandler import Email
from statistics import Statistics
from recipe import Recipe
@@ -145,23 +146,40 @@ class Updater(object):
self.machines = settings.get('machines', 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
self.upgrade_steps = [
+ (self._load_env, "Loading environment ..."),
(self._create_workdir, None),
- (self._get_env, "Loading environment ..."),
(self._detect_repo, "Detecting git repository location ..."),
(self._clean_repo, "Cleaning git repository of temporary branch ..."),
(self._detect_recipe_type, None),
+ (self._buildhistory_init, None),
(self._unpack_original, "Fetch & unpack original version ..."),
(self._rename, "Renaming recipes, reset PR (if exists) ..."),
(self._cleanall, "Clean all ..."),
(self._fetch, "Fetch new version (old checksums) ..."),
- (self._compile, None)
+ (self._compile, None),
+ (self._buildhistory_diff, None)
]
+ try:
+ self.base_env = self._get_env()
+ except EmptyEnvError as e:
+ import traceback
+ E( " %s\n%s" % (e.message, traceback.format_exc()))
+ E( " Bitbake output:\n%s" % (e.stdout))
+ exit(1)
+ self.buildhistory_enabled = self._buildhistory_is_enabled()
+
self.email_handler = Email(settings)
self.statistics = Statistics()
- def _get_env(self):
- stdout = self.bb.env(self.pn)
+ def _get_status_msg(self, err):
+ if err:
+ return str(err)
+ else:
+ return "Succeeded"
+
+ def _get_env(self, pn=None):
+ stdout = self.bb.env(pn)
assignment = re.compile("^([^ \t=]*)=(.*)")
bb_env = dict()
@@ -173,38 +191,46 @@ class Updater(object):
bb_env[m.group(1)] = m.group(2).strip("\"")
- self.env = bb_env
- self.recipe_dir = os.path.dirname(self.env['FILE'])
+ if not bb_env:
+ raise EmptyEnvError(stdout)
- def _detect_recipe_type(self):
- if self.env['SRC_URI'].find("ftp://") != -1 or \
- self.env['SRC_URI'].find("http://") != -1 or \
- self.env['SRC_URI'].find("https://") != -1:
- recipe = Recipe
- elif self.env['SRC_URI'].find("git://") != -1:
- recipe = GitRecipe
- else:
- raise UnsupportedProtocolError
+ return bb_env
- self.recipe = recipe(self.env, self.new_ver, self.interactive, self.workdir,
- self.recipe_dir, self.bb, self.git)
+ def _buildhistory_is_enabled(self):
+ enabled = False
- def _get_status_msg(self, err):
- if err:
- return str(err)
- else:
- return "Succeeded"
+ if 'buildhistory' in self.base_env['INHERIT']:
+ if not 'BUILDHISTORY_COMMIT' in self.base_env:
+ E(" Buildhistory was enabled but need"\
+ " BUILDHISTORY_COMMIT=1 please set.")
+ exit(1)
+
+ if not self.base_env['BUILDHISTORY_COMMIT'] == '1':
+ E(" Buildhistory was enabled but need"\
+ " BUILDHISTORY_COMMIT=1 please set.")
+ exit(1)
+
+ if self.skip_compilation:
+ W(" Buildhistory disabled because user" \
+ " skip compilation!")
+ else:
+ enabled = True
+
+ return enabled
+
+ def _load_env(self):
+ self.env = self._get_env(self.pn)
def _create_workdir(self):
self.workdir = os.path.join(self.uh_work_dir, self.pn)
- if not os.path.exists(self.workdir):
- os.mkdir(self.workdir)
- else:
- for f in os.listdir(self.workdir):
- os.remove(os.path.join(self.workdir, f))
+ if os.path.exists(self.workdir):
+ shutil.rmtree(self.workdir)
+ os.mkdir(self.workdir)
def _detect_repo(self):
+ self.recipe_dir = os.path.dirname(self.env['FILE'])
+
if self.env['PKGV'] == self.new_ver:
raise UpgradeNotNeededError
@@ -227,7 +253,7 @@ class Updater(object):
self.git.reset_hard()
self.git.clean_untracked()
- self._get_env()
+ self.env = self._get_env(self.pn)
def _clean_repo(self):
try:
@@ -239,14 +265,34 @@ class Updater(object):
except:
pass
+ def _detect_recipe_type(self):
+ if self.env['SRC_URI'].find("ftp://") != -1 or \
+ self.env['SRC_URI'].find("http://") != -1 or \
+ self.env['SRC_URI'].find("https://") != -1:
+ recipe = Recipe
+ elif self.env['SRC_URI'].find("git://") != -1:
+ recipe = GitRecipe
+ else:
+ raise UnsupportedProtocolError
+
+ self.recipe = recipe(self.env, self.new_ver, self.interactive, self.workdir,
+ self.recipe_dir, self.bb, self.git)
+
+ def _buildhistory_init(self):
+ if self.buildhistory_enabled == False:
+ return
+
+ self.buildhistory = BuildHistory(self.bb, self.pn, self.workdir)
+ I(" %s: Initial buildhistory for %s ..." % (self.pn, self.machines))
+ self.buildhistory.init(self.machines)
+
def _unpack_original(self):
self.recipe.unpack()
def _rename(self):
self.recipe.rename()
- # fetch new environment
- self._get_env()
+ self.env = self._get_env(self.pn)
self.recipe.update_env(self.env)
@@ -264,6 +310,15 @@ class Updater(object):
for machine in self.machines:
I(" %s: compiling for %s ..." % (self.pn, machine))
self.recipe.compile(machine)
+ if self.buildhistory is not None:
+ self.buildhistory.add()
+
+ def _buildhistory_diff(self):
+ if self.buildhistory_enabled == False:
+ return
+
+ I(" %s: Checking buildhistory ..." % self.pn)
+ self.buildhistory.diff()
def _get_packages_to_upgrade(self, packages=None):
if packages is None:
--
1.9.1
More information about the yocto
mailing list