[yocto] [yocto-autobuilder][PATCH v2] bin/buildlogger: add new script to aid SWAT process
Joshua Lock
joshua.g.lock at intel.com
Mon May 9 05:30:50 PDT 2016
buildlogger will be started with the autobuilder and, when correctly
configured, monitor the AB's JSON API for newly started builds. When one is
detected information about the build will be posted to the wiki.
Requires a ConfigParser (ini) style configuration file at
AB_BASE/etc/buildlogger.conf formatted as follows:
[wikiuser]
username = botuser
password = botuserpassword
[wiki]
pagetitle = BuildLog
Signed-off-by: Joshua Lock <joshua.g.lock at intel.com>
---
Changes since v1:
* Add example buildlogger conf file
* Only start buildlogger when BUILDLOG_TO_WIKI is True
* Move the wiki and builder api url's to the config file
.gitignore | 2 +
bin/buildlogger | 277 ++++++++++++++++++++++++++++++++++++++++
config/autobuilder.conf.example | 2 +
etc/buildlogger.conf.example | 8 ++
yocto-start-autobuilder | 9 ++
yocto-stop-autobuilder | 45 ++++---
6 files changed, 324 insertions(+), 19 deletions(-)
create mode 100755 bin/buildlogger
create mode 100644 etc/buildlogger.conf.example
diff --git a/.gitignore b/.gitignore
index 3f9505b..48c8a85 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@
###################################################
buildset-config
config/autobuilder.conf
+etc/buildlogger.conf
# Everything else #
###################
@@ -25,6 +26,7 @@ yocto-controller/controller.cfg
yocto-controller/state.sqlite
yocto-controller/twistd.log*
yocto-controller/buildbot.tac
+yocto-controller/logger.log
yocto-worker/build-appliance/build(newcommits)
yocto-worker/buildbot.tac
yocto-worker/janitor.log
diff --git a/bin/buildlogger b/bin/buildlogger
new file mode 100755
index 0000000..635d55f
--- /dev/null
+++ b/bin/buildlogger
@@ -0,0 +1,277 @@
+#!/usr/bin/env python3
+'''
+Created on May 5, 2016
+
+__author__ = "Joshua Lock"
+__copyright__ = "Copyright 2016, Intel Corporation"
+__credits__ = ["Joshua Lock"]
+__license__ = "GPL"
+__version__ = "2.0"
+__maintainer__ = "Joshua Lock"
+__email__ = "joshua.g.lock at intel.com"
+'''
+
+# We'd probably benefit from using some caching, but first we'd need the AB API
+# to include
+#
+# We can set repo url, branch & commit for a bunch of repositorys.
+# Do they all get built for nightly?
+
+try:
+ import configparser
+except ImportError:
+ import ConfigParser as configparser
+import json
+import os
+import requests
+import signal
+import sys
+import time
+
+abapi = ''
+# Wiki editing params
+un = ''
+pw = ''
+wikiapi = ''
+title = ''
+
+last_logged = ''
+# TODO: probably shouldn't write files in the same location as the script?
+cachefile = 'buildlogger.lastbuild'
+tmpfile = '/tmp/.buildlogger.pid'
+
+
+# Load configuration information from an ini
+def load_config(configfile):
+ global un
+ global pw
+ global title
+ global wikiapi
+ global abapi
+ success = False
+
+ if os.path.exists(configfile):
+ try:
+ config = configparser.ConfigParser()
+ config.read(configfile)
+ un = config.get('wiki', 'username')
+ pw = config.get('wiki', 'password')
+ title = config.get('wiki', 'pagetitle')
+ wikiapi = config.get('wiki', 'apiuri')
+ abapi = config.get('builder', 'apiuri')
+ success = True
+ except configparser.Error as ex:
+ print('Failed to load buildlogger configuration with error: %s' % str(ex))
+ else:
+ print('Config file %s does not exist, please create and populate it.' % configfile)
+
+ return success
+
+# we can't rely on the built in JSON parser in the requests module because
+# the JSON we get from the wiki begins with a UTF-8 BOM which chokes
+# json.loads().
+# Thus we decode the raw resonse content into a string and load that into a
+# JSON object ourselves.
+#
+# http://en.wikipedia.org/wiki/Byte_Order_Mark
+# http://bugs.python.org/issue18958
+def parse_json(response):
+ text = response.content.decode('utf-8-sig')
+
+ return json.loads(text)
+
+
+# Get the current content of the BuildLog page -- to make the wiki page as
+# useful as possible the most recent log entry should be at the top, to
+# that end we need to edit the whole page so that we can insert the new entry
+# after the log but before the other entries.
+# This method fetches the current page content, splits out the blurb and
+# returns a pair:
+# 1) the blurb
+# 2) the current entries
+def wiki_get_content():
+ params = '?format=json&action=query&prop=revisions&rvprop=content&titles='
+ req = requests.get(wikiapi+params+title)
+ parsed = parse_json(req)
+ pageid = sorted(parsed['query']['pages'].keys())[-1]
+ content = parsed['query']['pages'][pageid]['revisions'][0]['*']
+ blurb, entries = content.split('==', 1)
+ # ensure we keep only a single newline after the blurb
+ blurb = blurb.strip() + "\n"
+ entries = '=='+entries
+
+ return blurb, entries
+
+
+# Login to the wiki and return cookies for the logged in session
+def wiki_login():
+ payload = {
+ 'action': 'login',
+ 'lgname': un,
+ 'lgpassword': pw,
+ 'utf8': '',
+ 'format': 'json'
+ }
+ req1 = requests.post(wikiapi, data=payload)
+ parsed = parse_json(req1)
+ login_token = parsed['login']['token']
+
+ payload['lgtoken'] = login_token
+ req2 = requests.post(wikiapi, data=payload, cookies=req1.cookies)
+
+ return req2.cookies.copy()
+
+
+# Post the new page contents *content* with a summary of the action *summary*
+def wiki_post_page(content, summary, cookies):
+ params = '?format=json&action=query&prop=info|revisions&intoken=edit&rvprop=timestamp&titles='
+ req = requests.get(wikiapi+params+title, cookies=cookies)
+
+ parsed = parse_json(req)
+ pageid = sorted(parsed['query']['pages'].keys())[-1]
+ edit_token = parsed['query']['pages'][pageid]['edittoken']
+
+ edit_cookie = cookies.copy()
+ edit_cookie.update(req.cookies)
+
+ payload = {
+ 'action': 'edit',
+ 'assert': 'user',
+ 'title': title,
+ 'summary': summary,
+ 'text': content,
+ 'token': edit_token,
+ 'utf8': '',
+ 'format': 'json'
+ }
+
+ req = requests.post(wikiapi, data=payload, cookies=edit_cookie)
+ if not req.status_code == requests.codes.ok:
+ print("Unexpected status code %s received when trying to post entry to"
+ "the wiki." % req.status_code)
+ return False
+ else:
+ return True
+
+
+# Extract required info about the last build from the Autobuilder's JSON API
+# and format it for entry into the BuildLog, along with a summary of the edit
+def ab_last_build_to_entry(build_json, build_id):
+ build_info = build_json[build_id]
+ builder = build_info.get('builderName', 'Unknown builder')
+ reason = build_info.get('reason', 'No reason given')
+ buildid = build_info.get('number', '')
+ buildbranch = ''
+ chash = ''
+ for prop in build_info.get('properties'):
+ if prop[0] == 'branch':
+ buildbranch = prop[1]
+ # TODO: is it safe to assume we're building from the poky repo? Or at
+ # least only to log the poky commit hash.
+ if prop[0] == 'commit_poky':
+ chash = prop[1]
+
+ urlfmt = 'https://autobuilder.yoctoproject.org/main/builders/%s/builds/%s/'
+ url = urlfmt % (builder, buildid)
+ sectionfmt = '==[%s %s %s - %s %s]=='
+ section_title = sectionfmt % (url, builder, buildid, buildbranch, chash)
+ summaryfmt = 'Adding new BuildLog entry for build %s (%s)'
+ summary = summaryfmt % (buildid, chash)
+ content = "* '''Build ID''' - %s\n" % chash
+ content = content + '* ' + reason + '\n'
+ new_entry = '%s\n%s\n' % (section_title, content)
+
+ return new_entry, summary
+
+
+# Write the last logged build id to a file
+def write_last_build(buildid):
+ with open(cachefile, 'w') as fi:
+ fi.write(buildid)
+
+
+# Read last logged buildid from a file
+def read_last_build():
+ last_build = ''
+ try:
+ with open(cachefile, 'r') as fi:
+ last_build = fi.readline()
+ except FileNotFoundError as ex:
+ # A build hasn't been logged yet
+ pass
+ except Exception as e:
+ print('Error reading last build %s' % str(e))
+
+ return last_build
+
+
+def watch_for_builds(configfile):
+ if not load_config(configfile):
+ print('Failed to start buildlogger.')
+ sys.exit(1)
+ last_logged = read_last_build()
+
+ while True:
+ # wait a minute...
+ time.sleep(60)
+
+ builds = requests.get(abapi)
+
+ if not builds:
+ print("Failed to fetch Autobuilder data. Exiting.")
+ continue
+ try:
+ build_json = builds.json()
+ except Exception as e:
+ print("Failed to decode JSON: %s" % str(e))
+ continue
+
+ last_build = sorted(build_json.keys())[-1]
+ # If a new build is detected, post a new entry to the BuildLog
+ if last_build != last_logged:
+ new_entry, summary = ab_last_build_to_entry(build_json, last_build)
+ blurb, entries = wiki_get_content()
+ entries = new_entry+entries
+ cookies = wiki_login()
+ if wiki_post_page(blurb+entries, summary, cookies):
+ write_last_build(last_build)
+ last_logged = last_build
+ print("Entry posted:\n%s\n" % new_entry)
+ else:
+ print("Failed to post new entry.")
+
+ sys.exit(0)
+
+
+if __name__ == "__main__":
+ if len(sys.argv) < 2:
+ print('Please specify the path to the config file on the command line as the first argument.')
+ sys.exit(1)
+
+ # Check to see if this is running already. If so, kill it and rerun
+ if os.path.exists(tmpfile) and os.path.isfile(tmpfile):
+ print("A prior PID file exists. Attempting to kill.")
+ with open(tmpfile, 'r') as f:
+ pid=f.readline()
+ try:
+ os.kill(int(pid), signal.SIGKILL)
+ # We need to sleep for a second or two just to give the SIGKILL time
+ time.sleep(2)
+ except OSError as ex:
+ print("""We weren't able to kill the prior buildlogger. Trying again.""")
+ pass
+ # Check if the process that we killed is alive.
+ try:
+ os.kill(int(pid), 0)
+ except OSError as ex:
+ pass
+ elif os.path.exists(tmpfile) and not os.path.isfile(tmpfile):
+ raise Exception("""/tmp/.buildlogger.pid is a directory, remove it to continue.""")
+ try:
+ os.unlink(tmpfile)
+ except:
+ pass
+ with open(tmpfile, 'w') as f:
+ f.write(str(os.getpid()))
+
+ watch_for_builds(sys.argv[1])
diff --git a/config/autobuilder.conf.example b/config/autobuilder.conf.example
index 335b356..9179d14 100644
--- a/config/autobuilder.conf.example
+++ b/config/autobuilder.conf.example
@@ -81,3 +81,5 @@ QA_MAIL_CC = "buildcc at localhost"
QA_MAIL_BCC = "buildbcc at localhost"
QA_MAIL_SIG = "Multiline\nSig\nLine"
+[Buildlogger]
+BUILDLOG_TO_WIKI = False
diff --git a/etc/buildlogger.conf.example b/etc/buildlogger.conf.example
new file mode 100644
index 0000000..459961c
--- /dev/null
+++ b/etc/buildlogger.conf.example
@@ -0,0 +1,8 @@
+[wiki]
+username = BuildlogBotUser
+password = InsertPasswordHere
+pagetitle = BuildLogPageTitle
+apiuri = https://wiki.yoctoproject.org/wiki/api.php
+
+[builder]
+apiuri = https://autobuilder.yoctoproject.org/main/json/builders/nightly/builds/_all
diff --git a/yocto-start-autobuilder b/yocto-start-autobuilder
index 85b748d..9bc8838 100755
--- a/yocto-start-autobuilder
+++ b/yocto-start-autobuilder
@@ -72,6 +72,15 @@ if os.path.isfile(os.path.join(AB_BASE, ".setupdone")):
os.chdir(os.path.join(AB_BASE, "yocto-controller"))
subprocess.call(["make", "start"])
os.chdir(AB_BASE)
+ if os.environ["BUILDLOG_TO_WIKI"] == "True":
+ logger_log = open('yocto-controller/buildlogger.log', 'a')
+ logger_log.write('[ buildlogger started: %s ]\n' % datetime.datetime.now())
+ subprocess.Popen('python bin/buildlogger ' + os.path.join(AB_BASE, 'etc/buildlogger.conf'),
+ shell=True, stdin=None,
+ stdout=logger_log,
+ stderr=logger_log,
+ close_fds=True)
+ logger_log.close()
if sys.argv[1] == "worker" or sys.argv[1] == "both":
if os.environ["PRSERV_HOST"] and os.environ["PRSERV_HOST"] == "localhost":
diff --git a/yocto-stop-autobuilder b/yocto-stop-autobuilder
index a313b27..df5fd34 100755
--- a/yocto-stop-autobuilder
+++ b/yocto-stop-autobuilder
@@ -48,30 +48,18 @@ for section_name in parser.sections():
for name, value in parser.items(section_name):
os.environ[name.upper()] = value.strip('"').strip("'")
-if sys.argv[1] == "controller" or sys.argv[1] == "both":
- os.chdir(os.path.join(AB_BASE, "yocto-controller"))
- subprocess.call(["make", "stop"])
- os.chdir(AB_BASE)
-if sys.argv[1] == "worker" or sys.argv[1] == "both":
- if os.environ["PRSERV_HOST"] and os.environ["PRSERV_HOST"] == "localhost":
- os.chdir(AB_BASE)
- subprocess.call([os.path.join(AB_BASE, "ab-prserv"), "stop"])
-
- os.chdir(os.path.join(AB_BASE, "yocto-worker"))
- subprocess.call(["make", "stop"])
- os.chdir(AB_BASE)
- tmpfile = '/tmp/.buildworker-janitor'+os.getcwd().replace('/', '-')
- if os.path.exists(tmpfile) and os.path.isfile(tmpfile):
+def killpid(pidfile):
+ if os.path.exists(pidfile) and os.path.isfile(pidfile):
print("A prior PID file exists. Attempting to kill.")
- with open(tmpfile, 'r') as f:
+ with open(pidfile, 'r') as f:
pid=f.readline()
try:
os.kill(int(pid), signal.SIGKILL)
# We need to sleep for a second or two just to give the SIGKILL time
time.sleep(2)
except OSError as ex:
- print("""We weren't able to kill the prior buildworker-janitor. Trying again.""")
+ print("""We weren't able to kill the owner of %s, trying again.""" % pidfile)
pass
# Check if the process that we killed is alive.
try:
@@ -80,10 +68,29 @@ if sys.argv[1] == "worker" or sys.argv[1] == "both":
HINT:use signal.SIGKILL or signal.SIGABORT""")
except OSError as ex:
pass
- elif os.path.exists(tmpfile) and not os.path.isfile(tmpfile):
- raise Exception(tmpfile + """ is a directory. Remove it to continue.""")
+ elif os.path.exists(pidfile) and not os.path.isfile(pidfile):
+ raise Exception(pidfile + """ is a directory. Remove it to continue.""")
try:
- os.unlink(tmpfile)
+ os.unlink(pidfile)
except:
pass
+if sys.argv[1] == "controller" or sys.argv[1] == "both":
+ os.chdir(os.path.join(AB_BASE, "yocto-controller"))
+ subprocess.call(["make", "stop"])
+ os.chdir(AB_BASE)
+ tmpfile = '/tmp/.buildlogger.pid'
+ killpid(tmpfile)
+
+
+if sys.argv[1] == "worker" or sys.argv[1] == "both":
+ if os.environ["PRSERV_HOST"] and os.environ["PRSERV_HOST"] == "localhost":
+ os.chdir(AB_BASE)
+ subprocess.call([os.path.join(AB_BASE, "ab-prserv"), "stop"])
+
+ os.chdir(os.path.join(AB_BASE, "yocto-worker"))
+ subprocess.call(["make", "stop"])
+ os.chdir(AB_BASE)
+ tmpfile = '/tmp/.buildworker-janitor'+os.getcwd().replace('/', '-')
+ killpid(tmpfile)
+
--
2.5.5
More information about the yocto
mailing list