[yocto] [layerindex-web][PATCH 1/3] update: fix handling of moves outside of a layer

[Paul Eggleton]

If a file is moved (renamed) to a path outside of the layer, e.g.
another layer within a multi-layer repository, then we need to treat it
as a delete. Up until now we were updating the path and continuing, and
then the recipe was also picked up as an add in the other layer, leading
to duplicate recipe entries. I'd noticed these duplicates before but up
until now I'd thought that they were due to another bug we already
fixed, apparently not.

In order to remove these erroneous duplicate entries in existing
databases I have also added a layerindex/tools/fixup_duplicates.py
script. I've also made the -r/--reload option delete them as well.

Signed-off-by: Paul Eggleton <paul.eggleton at linux.intel.com>
---
 layerindex/tools/fixup_duplicates.py | 69 ++++++++++++++++++++++++++++
 layerindex/update_layer.py           | 27 +++++++----
 2 files changed, 87 insertions(+), 9 deletions(-)
 create mode 100644 layerindex/tools/fixup_duplicates.py

diff --git a/layerindex/tools/fixup_duplicates.py b/layerindex/tools/fixup_duplicates.py
new file mode 100644
index 00000000..5abe2c2d
--- /dev/null
+++ b/layerindex/tools/fixup_duplicates.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+# Fix recipes that were moved out
+#
+# Copyright (C) 2017 Intel Corporation
+# Author: Paul Eggleton <paul.eggleton at linux.intel.com>
+#
+# Licensed under the MIT license, see COPYING.MIT for details
+
+
+import sys
+import os
+
+sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
+
+import optparse
+import utils
+import logging
+
+class DryRunRollbackException(Exception):
+    pass
+
+logger = utils.logger_create('LayerIndexFixup')
+
+
+
+def main():
+    parser = optparse.OptionParser(
+        usage = """
+    %prog [options""")
+
+    parser.add_option("-n", "--dry-run",
+            help = "Don't write any data back to the database",
+            action="store_true", dest="dryrun")
+    parser.add_option("-d", "--debug",
+            help = "Enable debug output",
+            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
+    parser.add_option("-q", "--quiet",
+            help = "Hide all output except error messages",
+            action="store_const", const=logging.ERROR, dest="loglevel")
+
+    options, args = parser.parse_args(sys.argv)
+
+    utils.setup_django()
+    import settings
+    from layerindex.models import Recipe
+    from django.db import transaction
+
+    logger.setLevel(options.loglevel)
+
+    try:
+        with transaction.atomic():
+            #LayerBranch.objects.filter(layermaintainer__isnull=True).delete()
+            #LayerItem.objects.filter(layerbranch__isnull=True).filter(classic=False).delete()
+            #LayerItem.objects.filter(layerbranch__isnull=True).filter(classic=False).delete()
+            for recipe in Recipe.objects.filter(filepath__startswith='../'):
+                print('Deleting erroneous recipe %s %s' % (recipe.layerbranch, recipe))
+                recipe.delete()
+
+            if options.dryrun:
+                raise DryRunRollbackException()
+    except DryRunRollbackException:
+        pass
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py
index d34d8a59..2b39e8ef 100644
--- a/layerindex/update_layer.py
+++ b/layerindex/update_layer.py
@@ -502,6 +502,10 @@ def main():
                         if skip:
                             continue
                         if oldpath.startswith(subdir_start):
+                            if not newpath.startswith(subdir_start):
+                                logger.debug("Treating rename of %s to %s as a delete since new path is outside layer" % (oldpath, newpath))
+                                other_deletes.append(diffitem)
+                                continue
                             (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
                             (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
                             if oldtypename != newtypename:
@@ -684,16 +688,21 @@ def main():
                         # First, check which recipes still exist
                         layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
                         for v in layerrecipe_values:
-                            root = os.path.join(layerdir, v['filepath'])
-                            fullpath = os.path.join(root, v['filename'])
-                            preserve = True
-                            if os.path.exists(fullpath):
-                                for removedir in removedirs:
-                                    if fullpath.startswith(removedir):
-                                        preserve = False
-                                        break
-                            else:
+                            if v['filepath'].startswith('../'):
+                                # FIXME: These recipes were present due to a bug (not handling renames
+                                # to paths outside the layer) - this can be removed at some point in the future
                                 preserve = False
+                            else:
+                                root = os.path.join(layerdir, v['filepath'])
+                                fullpath = os.path.join(root, v['filename'])
+                                if os.path.exists(fullpath):
+                                    preserve = True
+                                    for removedir in removedirs:
+                                        if fullpath.startswith(removedir):
+                                            preserve = False
+                                            break
+                                else:
+                                    preserve = False
 
                             if preserve:
                                 # Recipe still exists, update it
-- 
2.17.1