Update --ignore plugin and improve multi-input performance

pyproject-scancode-toolkit-mini.toml

@@ -256,7 +256,6 @@ scancode-train-gibberish-model = "textcode.train_gibberish_model:train_gibberish
 # scancode_pre_scan is the entry point for pre_scan plugins executed before the
 # scans. See also plugincode.pre_scan module for details and doc.
 [project.entry-points.scancode_pre_scan]
-ignore = "scancode.plugin_ignore:ProcessIgnore"
 facet = "summarycode.facet:AddFacet"
 
 

pyproject-scancode-toolkit.toml

@@ -257,7 +257,6 @@ scancode-train-gibberish-model = "textcode.train_gibberish_model:train_gibberish
 # scancode_pre_scan is the entry point for pre_scan plugins executed before the
 # scans. See also plugincode.pre_scan module for details and doc.
 [project.entry-points.scancode_pre_scan]
-ignore = "scancode.plugin_ignore:ProcessIgnore"
 facet = "summarycode.facet:AddFacet"
 
 

pyproject.toml

@@ -262,7 +262,6 @@ scancode-train-gibberish-model = "textcode.train_gibberish_model:train_gibberish
 # scancode_pre_scan is the entry point for pre_scan plugins executed before the
 # scans. See also plugincode.pre_scan module for details and doc.
 [project.entry-points.scancode_pre_scan]
-ignore = "scancode.plugin_ignore:ProcessIgnore"
 facet = "summarycode.facet:AddFacet"
 
 

src/commoncode/resource.py

@@ -40,6 +40,7 @@
 from commoncode.datautils import List
 from commoncode.datautils import Mapping
 from commoncode.datautils import String
+from commoncode.fileset import is_included
 from commoncode.filetype import is_file as filetype_is_file
 from commoncode.filetype import is_special
 from commoncode.fileutils import as_posixpath
@@ -99,7 +100,7 @@ def skip_ignored(location):
     if TRACE_DEEP:
         logger_debug()
         logger_debug(
-            "Codebase.populate: walk: ignored loc:",
+            "Codebase.populate: walk: skip_ignored:",
             location,
             "ignored:",
             ignored(location),
@@ -110,6 +111,42 @@ def skip_ignored(location):
     return is_special(location) or ignored(location)
 
 
+def is_ignored(location,  includes=tuple(), excludes=tuple()):
+
+    excludes = {
+        pattern: 'User ignore: Supplied by --ignore' for pattern in excludes
+    }
+
+    includes = {
+        pattern: 'User include: Supplied by --include' for pattern in includes
+    }
+
+    included_from_options = is_included(
+        path=location,
+        includes=includes,
+        excludes=excludes,
+    )
+
+    if TRACE_DEEP:
+        logger_debug(
+            "Codebase.populate: walk: is_ignored:",
+            "is_ignored: location:",
+            location,
+            "included_from_options:",
+            included_from_options,
+            "skip_ignored",
+            skip_ignored(location)
+        )
+
+    if skip_ignored(location) or not included_from_options:
+        if TRACE_DEEP:
+            logger_debug("is_ignored: location:", location, "is_skipped",)
+
+        return True
+
+    return False
+
+
 def depth_walk(
     root_location,
     max_depth,
@@ -203,6 +240,8 @@ class Codebase:
     __slots__ = (
         "max_depth",
         "location",
+        "includes",
+        "ignores",
         "has_single_resource",
         "resource_attributes",
         "resource_class",
@@ -237,6 +276,8 @@ def __init__(
         max_in_memory=10000,
         max_depth=0,
         paths=tuple(),
+        ignores=tuple(),
+        includes=tuple(),
         *args,
         **kwargs,
     ):
@@ -299,6 +340,8 @@ def __init__(
 
         # finally populate
         self.paths = self._prepare_clean_paths(paths)
+        self.includes = self._prepare_clean_paths(includes)
+        self.ignores = ignores
         self._populate()
 
     def _prepare_clean_paths(self, paths=tuple()):
@@ -462,11 +505,17 @@ def _populate(self):
             return
 
         if self.paths:
-            return self._create_resources_from_paths(root=root, paths=self.paths)
+            # In case of a list of full paths, we create resources without walking
+            return self._create_resources_from_full_paths(root=root, paths=self.paths)
+            # In case we have multiple 
         else:
-            return self._create_resources_from_root(root=root)
+            return self._create_resources_from_root(
+                root=root,
+                includes=self.includes,
+                ignores=self.ignores,
+            )
 
-    def _create_resources_from_paths(self, root, paths):
+    def _create_resources_from_full_paths(self, root, paths):
         # without paths we iterate the provided paths. We report an error
         # if a path is missing on disk.
 
@@ -484,22 +533,21 @@ def _create_resources_from_paths(self, root, paths):
                 msg = f"ERROR: cannot populate codebase: path: {path!r} not found in {res_loc!r}"
                 self.errors.append(msg)
                 raise Exception(path, join(base_location, path))
-                continue
 
             # create all parents. The last parent is the one we want to use
             parent = root
             if TRACE:
-                logger_debug("Codebase._create_resources_from_paths: parent", parent)
+                logger_debug("Codebase._create_resources_from_full_paths: parent", parent)
             for parent_path in get_ancestor_paths(path, include_self=False):
                 if TRACE:
                     logger_debug(
-                        f"  Codebase._create_resources_from_paths: parent_path: {parent_path!r}"
+                        f"  Codebase._create_resources_from_full_paths: parent_path: {parent_path!r}"
                     )
                 if not parent_path:
                     continue
                 newpar = parents_by_path.get(parent_path)
                 if TRACE:
-                    logger_debug("  Codebase._create_resources_from_paths: newpar", repr(newpar))
+                    logger_debug("  Codebase._create_resources_from_full_paths: newpar", repr(newpar))
 
                 if not newpar:
                     newpar = self._get_or_create_resource(
@@ -510,7 +558,7 @@ def _create_resources_from_paths(self, root, paths):
                     )
                     if not newpar:
                         raise Exception(
-                            "ERROR: Codebase._create_resources_from_paths:"
+                            "ERROR: Codebase._create_resources_from_full_paths:"
                             f" cannot create parent for: {parent_path!r}"
                         )
                     parent = newpar
@@ -519,7 +567,7 @@ def _create_resources_from_paths(self, root, paths):
 
                     if TRACE:
                         logger_debug(
-                            f"  Codebase._create_resources_from_paths:",
+                            f"  Codebase._create_resources_from_full_paths:",
                             f"created newpar: {newpar!r}",
                         )
 
@@ -530,10 +578,10 @@ def _create_resources_from_paths(self, root, paths):
                 is_file=isfile(res_loc),
             )
             if TRACE:
-                logger_debug("Codebase._create_resources_from_paths: resource", res)
+                logger_debug("Codebase._create_resources_from_full_paths: resource", res)
 
-    def _create_resources_from_root(self, root):
-        # without paths we walks the root location top-down
+    def _create_resources_from_root(self, root, includes, ignores):
+        # without paths we walk the root location top-down
 
         # track resources parents by location during construction.
         # NOTE: this cannot exhaust memory on a large codebase, because we do
@@ -546,23 +594,48 @@ def err(_error):
                 f"ERROR: cannot populate codebase: {_error}\n{traceback.format_exc()}"
             )
 
-        # Walk over the directory and build the resource tree
-        for top, dirs, files in depth_walk(
-            root_location=root.location,
-            max_depth=self.max_depth,
-            error_handler=err,
-        ):
-            parent = parents_by_loc.pop(top)
-            for created in self._create_resources(
-                parent=parent,
-                top=top,
-                dirs=dirs,
-                files=files,
+        # ignore creating resources based on path patterns
+        skip_ignored = partial(is_ignored, excludes=ignores)
+
+        if TRACE_DEEP:
+            logger_debug(f"parents_by_loc: {parents_by_loc}, ignores: {ignores}, includes: {includes}")
+
+        # in the case of a single input location, walking starts from
+        # the root and only the root location
+        if not includes:
+            includes = [root.location]
+        else:
+            # create the directory resources between the common
+            # prefix and the included locations so that they are
+            # connected to the root
+            for created in self._create_resources_common_prefix_to_inputs(
+                root=root,
+                includes=includes,
             ):
-                # on the plain, bare FS, files cannot be parents
                 if not created.is_file:
                     parents_by_loc[created.location] = created
 
+        # we start walking through all the input locations
+        for included_location in includes:
+            # Walk over the directory and build the resource tree
+            for top, dirs, files in depth_walk(
+                root_location=included_location,
+                skip_ignored=skip_ignored,
+                max_depth=self.max_depth,
+                error_handler=err,
+            ):
+                parent = parents_by_loc.pop(top)
+                for created in self._create_resources(
+                    parent=parent,
+                    top=top,
+                    dirs=dirs,
+                    files=files,
+                    skip_ignored=skip_ignored,
+                ):
+                    # on the plain, bare FS, files cannot be parents
+                    if not created.is_file:
+                        parents_by_loc[created.location] = created
+
     def _create_resources(self, parent, top, dirs, files, skip_ignored=skip_ignored):
         """
         Create and yield ``files`` and ``dirs`` children Resources of a
@@ -575,6 +648,8 @@ def _create_resources(self, parent, top, dirs, files, skip_ignored=skip_ignored)
             for name in names:
                 location = join(top, name)
                 if skip_ignored(location):
+                    if TRACE_DEEP:
+                        logger_debug(f"_create_resources, depth_walk loop: ignored location: {location}")
                     continue
                 res = self._get_or_create_resource(
                     name=name,
@@ -585,6 +660,28 @@ def _create_resources(self, parent, top, dirs, files, skip_ignored=skip_ignored)
                     logger_debug("Codebase.create_resources:", res)
                 yield res
 
+    def _create_resources_common_prefix_to_inputs(self, root, includes):
+
+        if TRACE_DEEP:
+            logger_debug(f"_create_resources_common_prefix_to_inputs: root:{root.location}, includes: {includes}")
+
+        for included_path in includes:
+            _, _, extra_dir_path = included_path.rpartition(root.location)
+            extra_dirs = extra_dir_path.strip("/").split("/")
+            if TRACE_DEEP:
+                    logger_debug(f"_create_resources_common_prefix_to_inputs: root:{root.location}, includes: {includes}")
+
+            dir_resource = root
+            for dir_segment in extra_dirs:
+                dir_resource = self._get_or_create_resource(
+                    name=dir_segment,
+                    parent=dir_resource,
+                    is_file=False,
+                )
+                if TRACE:
+                    logger_debug("Codebase.create_resources:", dir_resource)
+                yield dir_resource
+
     def _create_root_resource(self):
         """
         Create and return the root Resource of this codebase.
@@ -1550,8 +1647,8 @@ def clean_path(path):
     Return a cleaned and normalized POSIX ``path``.
     """
     path = path or ""
-    # convert to posix and ensure we have no slash at both ends
-    path = posixpath_normpath(path.replace("\\", "/").strip("/"))
+    # convert to posix and ensure we have no slash at the end
+    path = posixpath_normpath(path.replace("\\", "/").rstrip("/"))
     if path == ".":
         path = ""
     return path

src/commoncode/testcase.py

@@ -93,7 +93,7 @@ class FileDrivenTesting(object):
 
     test_data_dir = None
 
-    def get_test_loc(self, test_path, copy=False, debug=False, must_exist=True):
+    def get_test_loc(self, test_path, copy=False, debug=False, must_exist=True, relative=False):
         """
         Given a `test_path` relative to the self.test_data_dir directory, return the
         location to a test file or directory for this path. Copy to a temp
@@ -129,6 +129,11 @@ def get_test_loc(self, test_path, copy=False, debug=False, must_exist=True):
                 # cleanup of VCS that could be left over from checkouts
                 self.remove_vcs(target_dir)
                 test_loc = target_dir
+
+        if relative:
+            _, _, rel_test_loc = test_loc.rpartition(os.getcwd())
+            return rel_test_loc.strip("/")
+
         return test_loc
 
     def get_temp_file(self, extension=None, dir_name="td", file_name="tf"):