[poky] [PATCH 1/1] Add a new task checklicense and fix some bugs in distro_check.py

Joshua Lock josh at linux.intel.com
Mon May 16 10:58:08 PDT 2011


On Mon, 2011-05-16 at 21:19 +0800, Mei Lei wrote:
> distro_check.py: Create a new function called create_log_file to reduce a lot of repeat code in distrodata.bbclass.
>                  We needn't to create log file in function save_distro_check_result, because the log file has been generated in check_eventhandler.
>                  Another bug is that we maybe access the /tmp/Meego-1.0 before we create this file.
>                  Add a judge statement to decide whether we need to create this file firstly.
> distrodata.bbclass: Add a new task checklicense to collect missing text license information.
>                     This can help package-report system to know how many recipes are missing license text.
> 
> Signed-off-by: Mei Lei <lei.mei at intel.com>

Arguably this should be more than one patch but I'm much happier with
this iteration, thanks.

Acked-by: Joshua Lock <josh at linux.intel.com>
> ---
>  meta/classes/distrodata.bbclass |  177 ++++++++++++++++++++++-----------------
>  meta/lib/oe/distro_check.py     |   27 ++++--
>  2 files changed, 119 insertions(+), 85 deletions(-)
> 
> diff --git a/meta/classes/distrodata.bbclass b/meta/classes/distrodata.bbclass
> index 905dad7..f24cff8 100644
> --- a/meta/classes/distrodata.bbclass
> +++ b/meta/classes/distrodata.bbclass
> @@ -4,19 +4,9 @@ addhandler distro_eventhandler
>  python distro_eventhandler() {
>  
>      if bb.event.getName(e) == "BuildStarted":
> -	"""initialize log files."""
> -	logpath = bb.data.getVar('LOG_DIR', e.data, 1)
> -	bb.utils.mkdirhier(logpath)
> -	logfile = os.path.join(logpath, "distrodata.%s.csv" % bb.data.getVar('DATETIME', e.data, 1))
> -	if not os.path.exists(logfile):
> -		slogfile = os.path.join(logpath, "distrodata.csv")
> -		if os.path.exists(slogfile):
> -			os.remove(slogfile)
> -		os.system("touch %s" % logfile)
> -		os.symlink(logfile, slogfile)
> -		bb.data.setVar('LOG_FILE', logfile, e.data)
> -
> -	lf = bb.utils.lockfile(logfile + ".lock")
> +	import oe.distro_check as dc
> +	logfile = dc.create_log_file(e.data, "distrodata.csv")
> +	lf = bb.utils.lockfile("%s.lock" % logfile)
>  	f = open(logfile, "a")
>  	f.write("Package,Description,Owner,License,ChkSum,Status,VerMatch,Version,Upsteam,Non-Update,Reason,Recipe Status\n")
>          f.close()
> @@ -33,9 +23,9 @@ python do_distrodata_np() {
>          bb.note("Package Name: %s" % pn)
>  
>          import oe.distro_check as dist_check
> -        tmpdir = bb.data.getVar('TMPDIR', d, 1)
> +        tmpdir = bb.data.getVar('TMPDIR', d, True)
>          distro_check_dir = os.path.join(tmpdir, "distro_check")
> -        datetime = bb.data.getVar('DATETIME', localdata, 1)
> +        datetime = bb.data.getVar('DATETIME', localdata, True)
>          dist_check.update_distro_data(distro_check_dir, datetime)
>  
>  	if pn.find("-native") != -1:
> @@ -111,15 +101,15 @@ python do_distrodata_np() {
>  addtask distrodata
>  do_distrodata[nostamp] = "1"
>  python do_distrodata() {
> -	logpath = bb.data.getVar('LOG_DIR', d, 1)
> +	logpath = bb.data.getVar('LOG_DIR', d, True)
>  	bb.utils.mkdirhier(logpath)
>  	logfile = os.path.join(logpath, "distrodata.csv")
>  
>          import oe.distro_check as dist_check
>  	localdata = bb.data.createCopy(d)
> -        tmpdir = bb.data.getVar('TMPDIR', d, 1)
> +        tmpdir = bb.data.getVar('TMPDIR', d, True)
>          distro_check_dir = os.path.join(tmpdir, "distro_check")
> -        datetime = bb.data.getVar('DATETIME', localdata, 1)
> +        datetime = bb.data.getVar('DATETIME', localdata, True)
>          dist_check.update_distro_data(distro_check_dir, datetime)
>  
>          pn = bb.data.getVar("PN", d, True)
> @@ -189,7 +179,7 @@ python do_distrodata() {
>          # do the comparison
>          result = dist_check.compare_in_distro_packages_list(distro_check_dir, localdata)
>  
> -	lf = bb.utils.lockfile(logfile + ".lock")
> +	lf = bb.utils.lockfile("%s.lock" % logfile)
>  	f = open(logfile, "a")
>  	f.write("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s," % \
>  		  (pname, pdesc, maintainer, plicense, pchksum, hasrstatus, vermatch, pcurver, pupver, noupdate, noupdate_reason, rstatus))
> @@ -211,19 +201,10 @@ do_distrodataall() {
>  addhandler checkpkg_eventhandler
>  python checkpkg_eventhandler() {
>      if bb.event.getName(e) == "BuildStarted":
> -	"""initialize log files."""
> -	logpath = bb.data.getVar('LOG_DIR', e.data, 1)
> -	bb.utils.mkdirhier(logpath)
> -	logfile = os.path.join(logpath, "checkpkg.%s.csv" % bb.data.getVar('DATETIME', e.data, 1))
> -	if not os.path.exists(logfile):
> -		slogfile = os.path.join(logpath, "checkpkg.csv")
> -		if os.path.exists(slogfile):
> -			os.remove(slogfile)
> -		os.system("touch %s" % logfile)
> -		os.symlink(logfile, slogfile)
> -		bb.data.setVar('LOG_FILE', logfile, e.data)
> -
> -	lf = bb.utils.lockfile(logfile + ".lock")
> +	import oe.distro_check as dc
> +	logfile = dc.create_log_file(e.data, "checkpkg.csv")
> +
> +	lf = bb.utils.lockfile("%s.lock" % logfile)
>  	f = open(logfile, "a")
>  	f.write("Package\tVersion\tUpver\tLicense\tSection\tHome\tRelease\tPriority\tDepends\tBugTracker\tPE\tDescription\tStatus\tTracking\tURI\tMAINTAINER\n")
>          f.close()
> @@ -304,7 +285,7 @@ python do_checkpkg() {
>  		Clear internal url cache as it's a temporary check. Not doing so will have 
>  		bitbake check url multiple times when looping through a single url
>  		"""
> -		fn = bb.data.getVar('FILE', d, 1)
> +		fn = bb.data.getVar('FILE', d, True)
>  		bb.fetch2.urldata_cache[fn] = {}
>  
>  		"""
> @@ -335,7 +316,7 @@ python do_checkpkg() {
>  	Return new version if success, or else error in "Errxxxx" style
>  	"""
>  	def check_new_dir(url, curver, d):
> -		pn = bb.data.getVar('PN', d, 1)
> +		pn = bb.data.getVar('PN', d, True)
>  		f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-1-" % pn)
>  		status = internal_fetch_wget(url, d, f)
>  		fhtml = f.read()
> @@ -394,7 +375,7 @@ python do_checkpkg() {
>  		"""possible to have no version in pkg name, such as spectrum-fw"""
>  		if not re.search("\d+", curname):
>  			return pcurver
> -		pn = bb.data.getVar('PN', d, 1)
> +		pn = bb.data.getVar('PN', d, True)
>  		f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-2-" % pn)
>  		status = internal_fetch_wget(url, d, f)
>  		fhtml = f.read()
> @@ -437,35 +418,35 @@ python do_checkpkg() {
>  		f.close()
>  		"""if host hasn't directory information, no need to save tmp file"""
>  		if status != "ErrHostNoDir" and re.match("Err", status):
> -			logpath = bb.data.getVar('LOG_DIR', d, 1)
> +			logpath = bb.data.getVar('LOG_DIR', d, True)
>  			os.system("cp %s %s/" % (f.name, logpath))
>  		os.unlink(f.name)
>  		return status
>  
>  	"""first check whether a uri is provided"""
> -	src_uri = bb.data.getVar('SRC_URI', d, 1)
> +	src_uri = bb.data.getVar('SRC_URI', d, True)
>  	if not src_uri:
>  		return
>  
>  	"""initialize log files."""
> -	logpath = bb.data.getVar('LOG_DIR', d, 1)
> +	logpath = bb.data.getVar('LOG_DIR', d, True)
>  	bb.utils.mkdirhier(logpath)
>  	logfile = os.path.join(logpath, "checkpkg.csv")
>  
>  	"""generate package information from .bb file"""
> -	pname = bb.data.getVar('PN', d, 1)
> -	pdesc = bb.data.getVar('DESCRIPTION', d, 1)
> -	pgrp = bb.data.getVar('SECTION', d, 1)
> -	pversion = bb.data.getVar('PV', d, 1)
> -	plicense = bb.data.getVar('LICENSE',d,1)
> -	psection = bb.data.getVar('SECTION',d,1)
> -	phome = bb.data.getVar('HOMEPAGE', d, 1)
> -	prelease = bb.data.getVar('PR',d,1)
> -	ppriority = bb.data.getVar('PRIORITY',d,1)
> -	pdepends = bb.data.getVar('DEPENDS',d,1)
> -	pbugtracker = bb.data.getVar('BUGTRACKER',d,1)
> -	ppe = bb.data.getVar('PE',d,1)
> -	psrcuri = bb.data.getVar('SRC_URI',d,1)
> +	pname = bb.data.getVar('PN', d, True)
> +	pdesc = bb.data.getVar('DESCRIPTION', d, True)
> +	pgrp = bb.data.getVar('SECTION', d, True)
> +	pversion = bb.data.getVar('PV', d, True)
> +	plicense = bb.data.getVar('LICENSE', d, True)
> +	psection = bb.data.getVar('SECTION', d, True)
> +	phome = bb.data.getVar('HOMEPAGE', d, True)
> +	prelease = bb.data.getVar('PR', d, True)
> +	ppriority = bb.data.getVar('PRIORITY', d, True)
> +	pdepends = bb.data.getVar('DEPENDS', d, True)
> +	pbugtracker = bb.data.getVar('BUGTRACKER', d, True)
> +	ppe = bb.data.getVar('PE', d, True)
> +	psrcuri = bb.data.getVar('SRC_URI', d, True)
>  
>  	found = 0
>  	for uri in src_uri.split():
> @@ -483,9 +464,9 @@ python do_checkpkg() {
>  
>  	(type, host, path, user, pswd, parm) = bb.decodeurl(uri)
>  	if type in ['http', 'https', 'ftp']:
> -		pcurver = bb.data.getVar('PV', d, 1)
> +		pcurver = bb.data.getVar('PV', d, True)
>  	else:
> -		pcurver = bb.data.getVar("SRCREV", d, 1)
> +		pcurver = bb.data.getVar("SRCREV", d, True)
>  
>  	if type in ['http', 'https', 'ftp']:
>  		newver = pcurver
> @@ -509,7 +490,7 @@ python do_checkpkg() {
>  			newver = check_new_dir(alturi, dirver, d)
>  			altpath = path
>  			if not re.match("Err", newver) and dirver != newver:
> -				altpath = altpath.replace(dirver, newver, 1)
> +				altpath = altpath.replace(dirver, newver, True)
>  				
>  		"""Now try to acquire all remote files in current directory"""
>  		if not re.match("Err", newver):
> @@ -625,7 +606,7 @@ python do_checkpkg() {
>  		pstatus += ":%s%s" % (host, path)
>  
>  	"""Read from manual distro tracking fields as alternative"""
> -	pmver = bb.data.getVar("RECIPE_LATEST_VERSION", d, 1)
> +	pmver = bb.data.getVar("RECIPE_LATEST_VERSION", d, True)
>  	if not pmver:
>  		pmver = "N/A"
>  		pmstatus = "ErrNoRecipeData"
> @@ -639,7 +620,7 @@ python do_checkpkg() {
>  	psrcuri = psrcuri.split()[0]
>  	pdepends = "".join(pdepends.split("\t"))
>  	pdesc = "".join(pdesc.split("\t"))
> -	lf = bb.utils.lockfile(logfile + ".lock")
> +	lf = bb.utils.lockfile("%s.lock" % logfile)
>  	f = open(logfile, "a")
>  	f.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % \
>  		  (pname,pversion,pupver,plicense,psection, phome,prelease, ppriority,pdepends,pbugtracker,ppe,pdesc,pstatus,pmver,psrcuri,maintainer))
> @@ -654,25 +635,12 @@ do_checkpkgall() {
>  	:
>  }
>  
> -#addhandler check_eventhandler
> -python check_eventhandler() {
> +addhandler distro_check_eventhandler
> +python distro_check_eventhandler() {
>      if bb.event.getName(e) == "BuildStarted":
> -        import oe.distro_check as dc
> -        tmpdir = bb.data.getVar('TMPDIR', e.data, 1)
> -        distro_check_dir = os.path.join(tmpdir, "distro_check")
> -        datetime = bb.data.getVar('DATETIME', e.data, 1)
>          """initialize log files."""
> -        logpath = bb.data.getVar('LOG_DIR', e.data, 1)
> -        bb.utils.mkdirhier(logpath)
> -        logfile = os.path.join(logpath, "distrocheck.%s.csv" % bb.data.getVar('DATETIME', e.data, 1))
> -        if not os.path.exists(logfile):
> -                slogfile = os.path.join(logpath, "distrocheck.csv")
> -                if os.path.exists(slogfile):
> -                        os.remove(slogfile)
> -                os.system("touch %s" % logfile)
> -                os.symlink(logfile, slogfile)
> -                bb.data.setVar('LOG_FILE', logfile, e.data)
> -
> +        import oe.distro_check as dc
> +        result_file = dc.create_log_file(e.data, "distrocheck.csv")
>      return
>  }
>  
> @@ -681,18 +649,23 @@ do_distro_check[nostamp] = "1"
>  python do_distro_check() {
>      """checks if the package is present in other public Linux distros"""
>      import oe.distro_check as dc
> +    import bb
> +    import shutil
>      localdata = bb.data.createCopy(d)
>      bb.data.update_data(localdata)
> -    tmpdir = bb.data.getVar('TMPDIR', d, 1)
> +    tmpdir = bb.data.getVar('TMPDIR', d, True)
>      distro_check_dir = os.path.join(tmpdir, "distro_check")
> -    datetime = bb.data.getVar('DATETIME', localdata, 1)
> +    logpath = bb.data.getVar('LOG_DIR', d, True)
> +    bb.utils.mkdirhier(logpath)
> +    result_file = os.path.join(logpath, "distrocheck.csv")
> +    datetime = bb.data.getVar('DATETIME', localdata, True)
>      dc.update_distro_data(distro_check_dir, datetime)
>  
>      # do the comparison
>      result = dc.compare_in_distro_packages_list(distro_check_dir, d)
>  
>      # save the results
> -    dc.save_distro_check_result(result, datetime, d)
> +    dc.save_distro_check_result(result, datetime, result_file, d)
>  }
>  
>  addtask distro_checkall after do_distro_check
> @@ -701,3 +674,55 @@ do_distro_checkall[nostamp] = "1"
>  do_distro_checkall() {
>  	:
>  }
> +#
> +#Check Missing License Text.
> +#Use this task to generate the missing license text data for pkg-report system,
> +#then we can search those recipes which license text isn't exsit in common-licenses directory
> +#
> +addhandler checklicense_eventhandler
> +python checklicense_eventhandler() {
> +    if bb.event.getName(e) == "BuildStarted":
> +        """initialize log files."""
> +        import oe.distro_check as dc
> +        logfile = dc.create_log_file(e.data, "missinglicense.csv")
> +        lf = bb.utils.lockfile("%s.lock" % logfile)
> +        f = open(logfile, "a")
> +        f.write("Package\tLicense\tMissingLicense\n")
> +        f.close()
> +        bb.utils.unlockfile(lf)
> +    return
> +}
> +
> +addtask checklicense
> +do_checklicense[nostamp] = "1"
> +python do_checklicense() {
> +    import os
> +    import bb
> +    import shutil
> +    logpath = bb.data.getVar('LOG_DIR', d, True)
> +    bb.utils.mkdirhier(logpath)
> +    pn = bb.data.getVar('PN', d, True)
> +    logfile = os.path.join(logpath, "missinglicense.csv")
> +    generic_directory = bb.data.getVar('COMMON_LICENSE_DIR', d, True)
> +    license_types = bb.data.getVar('LICENSE', d, True)
> +    for license_type in ((license_types.replace('+', '').replace('|', '&')
> +                          .replace('(', '').replace(')', '').replace(';', '')
> +                          .replace(',', '').replace(" ", "").split("&"))):
> +        if not os.path.isfile(os.path.join(generic_directory, license_type)):
> +            lf = bb.utils.lockfile("%s.lock" % logfile)
> +            f = open(logfile, "a")
> +            f.write("%s\t%s\t%s\n" % \
> +                (pn,license_types,license_type))
> +            f.close()
> +            bb.utils.unlockfile(lf)
> +    return
> +}
> +
> +addtask checklicenseall after do_checklicense
> +do_checklicenseall[recrdeptask] = "do_checklicense"
> +do_checklicenseall[nostamp] = "1"
> +do_checklicenseall() {
> +	:
> +}
> +
> +
> diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py
> index c85d4fb..55cdcad 100644
> --- a/meta/lib/oe/distro_check.py
> +++ b/meta/lib/oe/distro_check.py
> @@ -73,7 +73,8 @@ def clean_package_list(package_list):
>  def get_latest_released_meego_source_package_list():
>      "Returns list of all the name os packages in the latest meego distro"
>  
> -    
> +    if not os.path.isfile("/tmp/Meego-1.0"):
> +        os.mknod("/tmp/Meego-1.0")
>      f = open("/tmp/Meego-1.0", "r")
>      package_names = []
>      for line in f:
> @@ -341,7 +342,22 @@ def compare_in_distro_packages_list(distro_check_dir, d):
>      bb.note("Matching: %s" % matching_distros)
>      return matching_distros
>  
> -def save_distro_check_result(result, datetime, d):
> +def create_log_file(d, logname):
> +    logpath = bb.data.getVar('LOG_DIR', d, True)
> +    bb.utils.mkdirhier(logpath)
> +    logfn, logsuffix = os.path.splitext(logname)
> +    logfile = os.path.join(logpath, "%s.%s%s" % (logfn, bb.data.getVar('DATETIME', d, True), logsuffix))
> +    if not os.path.exists(logfile):
> +            slogfile = os.path.join(logpath, logname)
> +            if os.path.exists(slogfile):
> +                    os.remove(slogfile)
> +            os.system("touch %s" % logfile)
> +            os.symlink(logfile, slogfile)
> +            bb.data.setVar('LOG_FILE', logfile, d)
> +    return logfile
> +
> +
> +def save_distro_check_result(result, datetime, result_file, d):
>      pn = bb.data.getVar('PN', d, True)
>      logdir = bb.data.getVar('LOG_DIR', d, True)
>      if not logdir:
> @@ -349,16 +365,9 @@ def save_distro_check_result(result, datetime, d):
>          return
>      if not os.path.isdir(logdir):
>          os.makedirs(logdir)
> -    result_file = os.path.join(logdir, "distrocheck.%s.csv" % datetime)
>      line = pn
>      for i in result:
>          line = line + "," + i
> -    if not os.path.exists(result_file):
> -        sresult_file = os.path.join(logdir, "distrocheck.csv")
> -	if os.path.exists(sresult_file):
> -	    os.remove(sresult_file)
> -	os.system("touch %s" % result_file)
> -	os.symlink(result_file, sresult_file)
>      f = open(result_file, "a")
>      import fcntl
>      fcntl.lockf(f, fcntl.LOCK_EX)





More information about the poky mailing list