[yocto] [layerindex-web][PATCH v2 14/15] update_layer.py: use new-style transaction API
Paul Eggleton
paul.eggleton at linux.intel.com
Wed Jun 8 06:20:07 PDT 2016
The old transaction API has been removed in Django 1.8 and was
deprecated at 1.6. There's no explicit open transaction, commit or
rollback now - we just wrap the layer operations in
"with transaction.atomic()"; if we need to roll back we just raise a
"dummy" exception.
Signed-off-by: Paul Eggleton <paul.eggleton at linux.intel.com>
---
layerindex/update_layer.py | 781 ++++++++++++++++++++++-----------------------
1 file changed, 389 insertions(+), 392 deletions(-)
diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py
index bda79cb..fd8d3d4 100644
--- a/layerindex/update_layer.py
+++ b/layerindex/update_layer.py
@@ -34,6 +34,10 @@ except ImportError:
sys.exit(1)
+class DryRunRollbackException(Exception):
+ pass
+
+
def check_machine_conf(path, subdir_start):
subpath = path[len(subdir_start):]
res = conf_re.match(subpath)
@@ -187,429 +191,422 @@ def main():
# why won't they just fix that?!)
tinfoil.config_data.setVar('LICENSE', '')
- transaction.enter_transaction_management()
- transaction.managed(True)
try:
- layer = utils.get_layer(options.layer)
- urldir = layer.get_fetch_dir()
- repodir = os.path.join(fetchdir, urldir)
-
- layerbranch = layer.get_layerbranch(options.branch)
-
- branchname = options.branch
- branchdesc = options.branch
- if layerbranch:
- if layerbranch.actual_branch:
- branchname = layerbranch.actual_branch
- branchdesc = "%s (%s)" % (options.branch, branchname)
-
- # Collect repo info
- repo = git.Repo(repodir)
- assert repo.bare == False
- try:
- if options.nocheckout:
- topcommit = repo.commit('HEAD')
- else:
- topcommit = repo.commit('origin/%s' % branchname)
- except:
- if layerbranch:
- logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
- else:
- logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
- transaction.rollback()
- sys.exit(1)
-
- newbranch = False
- if not layerbranch:
- # LayerBranch doesn't exist for this branch, create it
- newbranch = True
- layerbranch = LayerBranch()
- layerbranch.layer = layer
- layerbranch.branch = branch
- layerbranch_source = layer.get_layerbranch('master')
- if not layerbranch_source:
- layerbranch_source = layer.get_layerbranch(None)
- if layerbranch_source:
- layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
- layerbranch.save()
- if layerbranch_source:
- for maintainer in layerbranch_source.layermaintainer_set.all():
- maintainer.pk = None
- maintainer.id = None
- maintainer.layerbranch = layerbranch
- maintainer.save()
- for dep in layerbranch_source.dependencies_set.all():
- dep.pk = None
- dep.id = None
- dep.layerbranch = layerbranch
- dep.save()
-
- if layerbranch.vcs_subdir and not options.nocheckout:
- # Find latest commit in subdirectory
- # A bit odd to do it this way but apparently there's no other way in the GitPython API
- topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
- if not topcommit:
- # This will error out if the directory is completely invalid or had never existed at this point
- # If it previously existed but has since been deleted, you will get the revision where it was
- # deleted - so we need to handle that case separately later
- if newbranch:
- logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
- elif layerbranch.vcs_subdir:
- logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
- else:
- logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
- transaction.rollback()
- sys.exit(1)
-
- layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
- layerdir_start = os.path.normpath(layerdir) + os.sep
- layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
- layermachines = Machine.objects.filter(layerbranch=layerbranch)
- layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
- layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
- if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
- # Check out appropriate branch
- if not options.nocheckout:
- out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
- out = utils.runcmd("git clean -f -x", repodir, logger=logger)
-
- if layerbranch.vcs_subdir and not os.path.exists(layerdir):
- if newbranch:
- logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
- else:
- logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
- transaction.rollback()
- sys.exit(1)
+ with transaction.atomic():
+ layer = utils.get_layer(options.layer)
+ urldir = layer.get_fetch_dir()
+ repodir = os.path.join(fetchdir, urldir)
- if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
- logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
- transaction.rollback()
- sys.exit(1)
+ layerbranch = layer.get_layerbranch(options.branch)
- logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
+ branchname = options.branch
+ branchdesc = options.branch
+ if layerbranch:
+ if layerbranch.actual_branch:
+ branchname = layerbranch.actual_branch
+ branchdesc = "%s (%s)" % (options.branch, branchname)
+ # Collect repo info
+ repo = git.Repo(repodir)
+ assert repo.bare == False
try:
- config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
- except recipeparse.RecipeParseError as e:
- logger.error(str(e))
- transaction.rollback()
+ if options.nocheckout:
+ topcommit = repo.commit('HEAD')
+ else:
+ topcommit = repo.commit('origin/%s' % branchname)
+ except:
+ if layerbranch:
+ logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
+ else:
+ logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
sys.exit(1)
- if layerbranch.vcs_last_rev and not options.reload:
+ newbranch = False
+ if not layerbranch:
+ # LayerBranch doesn't exist for this branch, create it
+ newbranch = True
+ layerbranch = LayerBranch()
+ layerbranch.layer = layer
+ layerbranch.branch = branch
+ layerbranch_source = layer.get_layerbranch('master')
+ if not layerbranch_source:
+ layerbranch_source = layer.get_layerbranch(None)
+ if layerbranch_source:
+ layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
+ layerbranch.save()
+ if layerbranch_source:
+ for maintainer in layerbranch_source.layermaintainer_set.all():
+ maintainer.pk = None
+ maintainer.id = None
+ maintainer.layerbranch = layerbranch
+ maintainer.save()
+ for dep in layerbranch_source.dependencies_set.all():
+ dep.pk = None
+ dep.id = None
+ dep.layerbranch = layerbranch
+ dep.save()
+
+ if layerbranch.vcs_subdir and not options.nocheckout:
+ # Find latest commit in subdirectory
+ # A bit odd to do it this way but apparently there's no other way in the GitPython API
+ topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
+ if not topcommit:
+ # This will error out if the directory is completely invalid or had never existed at this point
+ # If it previously existed but has since been deleted, you will get the revision where it was
+ # deleted - so we need to handle that case separately later
+ if newbranch:
+ logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
+ elif layerbranch.vcs_subdir:
+ logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
+ else:
+ logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
+ sys.exit(1)
+
+ layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
+ layerdir_start = os.path.normpath(layerdir) + os.sep
+ layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
+ layermachines = Machine.objects.filter(layerbranch=layerbranch)
+ layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
+ layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
+ if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
+ # Check out appropriate branch
+ if not options.nocheckout:
+ out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
+ out = utils.runcmd("git clean -f -x", repodir, logger=logger)
+
+ if layerbranch.vcs_subdir and not os.path.exists(layerdir):
+ if newbranch:
+ logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
+ else:
+ logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
+ sys.exit(1)
+
+ if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
+ logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
+ sys.exit(1)
+
+ logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
+
try:
- diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
- except Exception as e:
- logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
- diff = None
- else:
- diff = None
-
- # We handle recipes specially to try to preserve the same id
- # when recipe upgrades happen (so that if a user bookmarks a
- # recipe page it remains valid)
- layerrecipes_delete = []
- layerrecipes_add = []
-
- # Check if any paths should be ignored because there are layers within this layer
- removedirs = []
- for root, dirs, files in os.walk(layerdir):
- for diritem in dirs:
- if os.path.exists(os.path.join(root, diritem, 'conf', 'layer.conf')):
- removedirs.append(os.path.join(root, diritem) + os.sep)
-
- if diff:
- # Apply git changes to existing recipe list
-
- if layerbranch.vcs_subdir:
- subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
+ config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
+ except recipeparse.RecipeParseError as e:
+ logger.error(str(e))
+ sys.exit(1)
+
+ if layerbranch.vcs_last_rev and not options.reload:
+ try:
+ diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
+ except Exception as e:
+ logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
+ diff = None
else:
- subdir_start = ""
-
- updatedrecipes = set()
- dirtyrecipes = set()
- other_deletes = []
- other_adds = []
- for diffitem in diff.iter_change_type('R'):
- oldpath = diffitem.a_blob.path
- newpath = diffitem.b_blob.path
- skip = False
- for removedir in removedirs:
- # FIXME what about files moved into removedirs?
- if oldpath.startswith(removedir):
- skip = True
- break
- if skip:
- continue
- if oldpath.startswith(subdir_start):
- (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
- (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
- if oldtypename != newtypename:
- # This is most likely to be a .inc file renamed to a .bb - and since
- # there may be another recipe deleted at the same time we probably want
- # to consider that, so just treat it as a delete and an add
- logger.debug("Treating rename of %s to %s as a delete and add (since type changed)" % (oldpath, newpath))
- other_deletes.append(diffitem)
- other_adds.append(diffitem)
- elif oldtypename == 'recipe':
- results = layerrecipes.filter(filepath=oldfilepath).filter(filename=oldfilename)
- if len(results):
- recipe = results[0]
- logger.debug("Rename recipe %s to %s" % (recipe, newpath))
- recipe.filepath = newfilepath
- recipe.filename = newfilename
- recipe.save()
- update_recipe_file(config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir)
- updatedrecipes.add(os.path.join(oldfilepath, oldfilename))
- updatedrecipes.add(os.path.join(newfilepath, newfilename))
- else:
- logger.warn("Renamed recipe %s could not be found" % oldpath)
- other_adds.append(diffitem)
- elif oldtypename == 'bbappend':
- results = layerappends.filter(filepath=oldfilepath).filter(filename=oldfilename)
- if len(results):
- logger.debug("Rename bbappend %s to %s" % (results[0], newfilepath))
- results[0].filepath = newfilepath
- results[0].filename = newfilename
- results[0].save()
- else:
- logger.warn("Renamed bbappend %s could not be found" % oldpath)
- other_adds.append(diffitem)
- elif oldtypename == 'machine':
- results = layermachines.filter(name=oldfilename)
- if len(results):
- logger.debug("Rename machine %s to %s" % (results[0], newfilename))
- results[0].name = newfilename
- results[0].save()
- else:
- logger.warn("Renamed machine %s could not be found" % oldpath)
- other_adds.append(diffitem)
- elif oldtypename == 'bbclass':
- results = layerclasses.filter(name=oldfilename)
- if len(results):
- logger.debug("Rename class %s to %s" % (results[0], newfilename))
- results[0].name = newfilename
- results[0].save()
- else:
- logger.warn("Renamed class %s could not be found" % oldpath)
- other_adds.append(diffitem)
-
- deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=oldpath)
- for dep in deps:
- dirtyrecipes.add(dep.recipe)
+ diff = None
+ # We handle recipes specially to try to preserve the same id
+ # when recipe upgrades happen (so that if a user bookmarks a
+ # recipe page it remains valid)
+ layerrecipes_delete = []
+ layerrecipes_add = []
- for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes):
- path = diffitem.a_blob.path
- if path.startswith(subdir_start):
+ # Check if any paths should be ignored because there are layers within this layer
+ removedirs = []
+ for root, dirs, files in os.walk(layerdir):
+ for diritem in dirs:
+ if os.path.exists(os.path.join(root, diritem, 'conf', 'layer.conf')):
+ removedirs.append(os.path.join(root, diritem) + os.sep)
+
+ if diff:
+ # Apply git changes to existing recipe list
+
+ if layerbranch.vcs_subdir:
+ subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
+ else:
+ subdir_start = ""
+
+ updatedrecipes = set()
+ dirtyrecipes = set()
+ other_deletes = []
+ other_adds = []
+ for diffitem in diff.iter_change_type('R'):
+ oldpath = diffitem.a_blob.path
+ newpath = diffitem.b_blob.path
skip = False
for removedir in removedirs:
- if path.startswith(removedir):
+ # FIXME what about files moved into removedirs?
+ if oldpath.startswith(removedir):
skip = True
break
if skip:
continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
- if len(values):
- layerrecipes_delete.append(values[0])
- logger.debug("Mark %s for deletion" % values[0])
- updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
+ if oldpath.startswith(subdir_start):
+ (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
+ (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
+ if oldtypename != newtypename:
+ # This is most likely to be a .inc file renamed to a .bb - and since
+ # there may be another recipe deleted at the same time we probably want
+ # to consider that, so just treat it as a delete and an add
+ logger.debug("Treating rename of %s to %s as a delete and add (since type changed)" % (oldpath, newpath))
+ other_deletes.append(diffitem)
+ other_adds.append(diffitem)
+ elif oldtypename == 'recipe':
+ results = layerrecipes.filter(filepath=oldfilepath).filter(filename=oldfilename)
+ if len(results):
+ recipe = results[0]
+ logger.debug("Rename recipe %s to %s" % (recipe, newpath))
+ recipe.filepath = newfilepath
+ recipe.filename = newfilename
+ recipe.save()
+ update_recipe_file(config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir)
+ updatedrecipes.add(os.path.join(oldfilepath, oldfilename))
+ updatedrecipes.add(os.path.join(newfilepath, newfilename))
+ else:
+ logger.warn("Renamed recipe %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+ elif oldtypename == 'bbappend':
+ results = layerappends.filter(filepath=oldfilepath).filter(filename=oldfilename)
+ if len(results):
+ logger.debug("Rename bbappend %s to %s" % (results[0], newfilepath))
+ results[0].filepath = newfilepath
+ results[0].filename = newfilename
+ results[0].save()
+ else:
+ logger.warn("Renamed bbappend %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+ elif oldtypename == 'machine':
+ results = layermachines.filter(name=oldfilename)
+ if len(results):
+ logger.debug("Rename machine %s to %s" % (results[0], newfilename))
+ results[0].name = newfilename
+ results[0].save()
+ else:
+ logger.warn("Renamed machine %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+ elif oldtypename == 'bbclass':
+ results = layerclasses.filter(name=oldfilename)
+ if len(results):
+ logger.debug("Rename class %s to %s" % (results[0], newfilename))
+ results[0].name = newfilename
+ results[0].save()
+ else:
+ logger.warn("Renamed class %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+
+ deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=oldpath)
+ for dep in deps:
+ dirtyrecipes.add(dep.recipe)
+
+
+ for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes):
+ path = diffitem.a_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for removedir in removedirs:
+ if path.startswith(removedir):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
+ if len(values):
+ layerrecipes_delete.append(values[0])
+ logger.debug("Mark %s for deletion" % values[0])
+ updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
+ else:
+ logger.warn("Deleted recipe %s could not be found" % path)
+ elif typename == 'bbappend':
+ layerappends.filter(filepath=filepath).filter(filename=filename).delete()
+ elif typename == 'machine':
+ layermachines.filter(name=filename).delete()
+ elif typename == 'bbclass':
+ layerclasses.filter(name=filename).delete()
+
+ for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds):
+ path = diffitem.b_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for removedir in removedirs:
+ if path.startswith(removedir):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ layerrecipes_add.append(os.path.join(repodir, path))
+ logger.debug("Mark %s for addition" % path)
+ updatedrecipes.add(os.path.join(filepath, filename))
+ elif typename == 'bbappend':
+ append = BBAppend()
+ append.layerbranch = layerbranch
+ append.filename = filename
+ append.filepath = filepath
+ append.save()
+ elif typename == 'machine':
+ machine = Machine()
+ machine.layerbranch = layerbranch
+ machine.name = filename
+ update_machine_conf_file(os.path.join(repodir, path), machine)
+ machine.save()
+ elif typename == 'bbclass':
+ bbclass = BBClass()
+ bbclass.layerbranch = layerbranch
+ bbclass.name = filename
+ bbclass.save()
+
+ for diffitem in diff.iter_change_type('M'):
+ path = diffitem.a_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for removedir in removedirs:
+ if path.startswith(removedir):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ logger.debug("Mark %s for update" % path)
+ results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
+ if results:
+ recipe = results[0]
+ update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
+ recipe.save()
+ updatedrecipes.add(recipe.full_path())
+ elif typename == 'machine':
+ results = layermachines.filter(name=filename)
+ if results:
+ machine = results[0]
+ update_machine_conf_file(os.path.join(repodir, path), machine)
+ machine.save()
+
+ deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
+ for dep in deps:
+ dirtyrecipes.add(dep.recipe)
+
+ for recipe in dirtyrecipes:
+ if not recipe.full_path() in updatedrecipes:
+ update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
+ else:
+ # Collect recipe data from scratch
+
+ layerrecipe_fns = []
+ if options.fullreload:
+ layerrecipes.delete()
+ else:
+ # First, check which recipes still exist
+ layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
+ for v in layerrecipe_values:
+ root = os.path.join(layerdir, v['filepath'])
+ fullpath = os.path.join(root, v['filename'])
+ preserve = True
+ if os.path.exists(fullpath):
+ for removedir in removedirs:
+ if fullpath.startswith(removedir):
+ preserve = False
+ break
else:
- logger.warn("Deleted recipe %s could not be found" % path)
- elif typename == 'bbappend':
- layerappends.filter(filepath=filepath).filter(filename=filename).delete()
- elif typename == 'machine':
- layermachines.filter(name=filename).delete()
- elif typename == 'bbclass':
- layerclasses.filter(name=filename).delete()
-
- for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds):
- path = diffitem.b_blob.path
- if path.startswith(subdir_start):
- skip = False
- for removedir in removedirs:
- if path.startswith(removedir):
- skip = True
- break
- if skip:
- continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- layerrecipes_add.append(os.path.join(repodir, path))
- logger.debug("Mark %s for addition" % path)
- updatedrecipes.add(os.path.join(filepath, filename))
- elif typename == 'bbappend':
- append = BBAppend()
- append.layerbranch = layerbranch
- append.filename = filename
- append.filepath = filepath
- append.save()
- elif typename == 'machine':
- machine = Machine()
- machine.layerbranch = layerbranch
- machine.name = filename
- update_machine_conf_file(os.path.join(repodir, path), machine)
- machine.save()
- elif typename == 'bbclass':
- bbclass = BBClass()
- bbclass.layerbranch = layerbranch
- bbclass.name = filename
- bbclass.save()
-
- for diffitem in diff.iter_change_type('M'):
- path = diffitem.a_blob.path
- if path.startswith(subdir_start):
- skip = False
- for removedir in removedirs:
- if path.startswith(removedir):
- skip = True
- break
- if skip:
- continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- logger.debug("Mark %s for update" % path)
- results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
- if results:
+ preserve = False
+
+ if preserve:
+ # Recipe still exists, update it
+ results = layerrecipes.filter(id=v['id'])[:1]
recipe = results[0]
- update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
- recipe.save()
- updatedrecipes.add(recipe.full_path())
- elif typename == 'machine':
- results = layermachines.filter(name=filename)
- if results:
- machine = results[0]
- update_machine_conf_file(os.path.join(repodir, path), machine)
+ update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
+ else:
+ # Recipe no longer exists, mark it for later on
+ layerrecipes_delete.append(v)
+ layerrecipe_fns.append(fullpath)
+
+ layermachines.delete()
+ layerappends.delete()
+ layerclasses.delete()
+ for root, dirs, files in os.walk(layerdir):
+ if '.git' in dirs:
+ dirs.remove('.git')
+ for diritem in dirs[:]:
+ fullpath = os.path.join(root, diritem) + os.sep
+ if fullpath in removedirs:
+ dirs.remove(diritem)
+ for f in files:
+ fullpath = os.path.join(root, f)
+ (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
+ if typename == 'recipe':
+ if fullpath not in layerrecipe_fns:
+ layerrecipes_add.append(fullpath)
+ elif typename == 'bbappend':
+ append = BBAppend()
+ append.layerbranch = layerbranch
+ append.filename = f
+ append.filepath = os.path.relpath(root, layerdir)
+ append.save()
+ elif typename == 'machine':
+ machine = Machine()
+ machine.layerbranch = layerbranch
+ machine.name = filename
+ update_machine_conf_file(fullpath, machine)
machine.save()
+ elif typename == 'bbclass':
+ bbclass = BBClass()
+ bbclass.layerbranch = layerbranch
+ bbclass.name = filename
+ bbclass.save()
+
+ for added in layerrecipes_add:
+ # This is good enough without actually parsing the file
+ (pn, pv) = split_recipe_fn(added)
+ oldid = -1
+ for deleted in layerrecipes_delete:
+ if deleted['pn'] == pn:
+ oldid = deleted['id']
+ layerrecipes_delete.remove(deleted)
+ break
+ if oldid > -1:
+ # Reclaim a record we would have deleted
+ results = Recipe.objects.filter(id=oldid)[:1]
+ recipe = results[0]
+ logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
+ else:
+ # Create new record
+ logger.debug("Add new recipe %s" % added)
+ recipe = Recipe()
+ recipe.layerbranch = layerbranch
+ recipe.filename = os.path.basename(added)
+ root = os.path.dirname(added)
+ recipe.filepath = os.path.relpath(root, layerdir)
+ update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
+ recipe.save()
- deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
- for dep in deps:
- dirtyrecipes.add(dep.recipe)
+ for deleted in layerrecipes_delete:
+ logger.debug("Delete %s" % deleted)
+ results = Recipe.objects.filter(id=deleted['id'])[:1]
+ recipe = results[0]
+ recipe.delete()
- for recipe in dirtyrecipes:
- if not recipe.full_path() in updatedrecipes:
- update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
+ # Save repo info
+ layerbranch.vcs_last_rev = topcommit.hexsha
+ layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
else:
- # Collect recipe data from scratch
+ logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
+
+ layerbranch.vcs_last_fetch = datetime.now()
+ layerbranch.save()
+
+ if options.dryrun:
+ raise DryRunRollbackException()
- layerrecipe_fns = []
- if options.fullreload:
- layerrecipes.delete()
- else:
- # First, check which recipes still exist
- layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
- for v in layerrecipe_values:
- root = os.path.join(layerdir, v['filepath'])
- fullpath = os.path.join(root, v['filename'])
- preserve = True
- if os.path.exists(fullpath):
- for removedir in removedirs:
- if fullpath.startswith(removedir):
- preserve = False
- break
- else:
- preserve = False
-
- if preserve:
- # Recipe still exists, update it
- results = layerrecipes.filter(id=v['id'])[:1]
- recipe = results[0]
- update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
- else:
- # Recipe no longer exists, mark it for later on
- layerrecipes_delete.append(v)
- layerrecipe_fns.append(fullpath)
-
- layermachines.delete()
- layerappends.delete()
- layerclasses.delete()
- for root, dirs, files in os.walk(layerdir):
- if '.git' in dirs:
- dirs.remove('.git')
- for diritem in dirs[:]:
- fullpath = os.path.join(root, diritem) + os.sep
- if fullpath in removedirs:
- dirs.remove(diritem)
- for f in files:
- fullpath = os.path.join(root, f)
- (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
- if typename == 'recipe':
- if fullpath not in layerrecipe_fns:
- layerrecipes_add.append(fullpath)
- elif typename == 'bbappend':
- append = BBAppend()
- append.layerbranch = layerbranch
- append.filename = f
- append.filepath = os.path.relpath(root, layerdir)
- append.save()
- elif typename == 'machine':
- machine = Machine()
- machine.layerbranch = layerbranch
- machine.name = filename
- update_machine_conf_file(fullpath, machine)
- machine.save()
- elif typename == 'bbclass':
- bbclass = BBClass()
- bbclass.layerbranch = layerbranch
- bbclass.name = filename
- bbclass.save()
-
- for added in layerrecipes_add:
- # This is good enough without actually parsing the file
- (pn, pv) = split_recipe_fn(added)
- oldid = -1
- for deleted in layerrecipes_delete:
- if deleted['pn'] == pn:
- oldid = deleted['id']
- layerrecipes_delete.remove(deleted)
- break
- if oldid > -1:
- # Reclaim a record we would have deleted
- results = Recipe.objects.filter(id=oldid)[:1]
- recipe = results[0]
- logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
- else:
- # Create new record
- logger.debug("Add new recipe %s" % added)
- recipe = Recipe()
- recipe.layerbranch = layerbranch
- recipe.filename = os.path.basename(added)
- root = os.path.dirname(added)
- recipe.filepath = os.path.relpath(root, layerdir)
- update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
- recipe.save()
-
- for deleted in layerrecipes_delete:
- logger.debug("Delete %s" % deleted)
- results = Recipe.objects.filter(id=deleted['id'])[:1]
- recipe = results[0]
- recipe.delete()
-
- # Save repo info
- layerbranch.vcs_last_rev = topcommit.hexsha
- layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
- else:
- logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
-
- layerbranch.vcs_last_fetch = datetime.now()
- layerbranch.save()
-
- if options.dryrun:
- transaction.rollback()
- else:
- transaction.commit()
except KeyboardInterrupt:
- transaction.rollback()
logger.warn("Update interrupted, changes to %s rolled back" % layer.name)
sys.exit(254)
+ except SystemExit:
+ raise
+ except DryRunRollbackException:
+ pass
except:
import traceback
traceback.print_exc()
- transaction.rollback()
- finally:
- transaction.leave_transaction_management()
shutil.rmtree(tempdir)
sys.exit(0)
--
2.5.5
More information about the yocto
mailing list