Keep temp cleanup failures from stranding the reports

src/fosslight_scanner/fosslight_scanner.py

@@ -10,6 +10,7 @@
 import warnings
 import shutil
 import shlex
+import stat
 import subprocess
 import platform
 from pathlib import Path
@@ -62,6 +63,38 @@
 ]
 
 
+def _add_write_permission(target):
+    # Add the owner write bit to the existing mode. Replacing the mode instead would
+    # drop the read and execute bits a directory needs, and os.walk() could no longer
+    # descend into it.
+    try:
+        os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE)
+    except OSError:
+        pass
+
+
+def remove_tree(path):
+    if not os.path.exists(path):
+        return True
+    try:
+        shutil.rmtree(path)
+        return True
+    except Exception as ex:
+        logger.debug(f"Retry to remove {path} after clearing read-only flags: {ex}")
+
+    try:
+        # os.walk() is top-down, so a directory is made writable before it is entered.
+        _add_write_permission(path)
+        for root, dirs, files in os.walk(path):
+            for name in dirs + files:
+                _add_write_permission(os.path.join(root, name))
+        shutil.rmtree(path)
+        return True
+    except Exception as ex:
+        logger.warning(f"Failed to remove the temporary directory({path}): {ex}")
+        return False
+
+
 def _all_exclude_mode_for_scanner(
         excluded_path_with_default_exclusion,
         excluded_path_without_dot,
@@ -539,7 +572,7 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
                                                 kb_token, binary_simple, formats, recursive_dep, no_merge)
 
                 if extract_folder:
-                    shutil.rmtree(extract_folder)
+                    remove_tree(extract_folder)
             else:
                 logger.error("(mode) No mode has been selected for analysis.")
         try:
@@ -553,22 +586,44 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
             except Exception as ex:
                 logger.debug(f"Failed to move log file: {ex}")
 
+            raw_data_removed = True
             if not keep_raw_data:
                 logger.debug(f"Remove temporary files: {_output_dir}")
-                shutil.rmtree(_output_dir)
+                raw_data_removed = remove_tree(_output_dir)
             if os.path.exists(output_path):
                 os.makedirs(final_dir, exist_ok=True)
+                raw_data_name = os.path.basename(os.path.normpath(_output_dir))
+                move_failed = False
                 for item in os.listdir(output_path):
+                    if not raw_data_removed and item == raw_data_name:
+                        # The raw data was meant to be discarded and could not be removed;
+                        # publishing it into the user's output directory is not the answer.
+                        continue
                     src_item = os.path.join(output_path, item)
                     dst_item = os.path.join(final_dir, item)
-                    if os.path.isdir(src_item) and os.path.exists(dst_item):
-                        for sub_item in os.listdir(src_item):
-                            shutil.move(os.path.join(src_item, sub_item), os.path.join(dst_item, sub_item))
-                    else:
-                        shutil.move(src_item, dst_item)
-                shutil.rmtree(output_path)
+                    try:
+                        if os.path.isdir(src_item) and os.path.exists(dst_item):
+                            for sub_item in os.listdir(src_item):
+                                shutil.move(os.path.join(src_item, sub_item), os.path.join(dst_item, sub_item))
+                        else:
+                            shutil.move(src_item, dst_item)
+                    except Exception as ex:
+                        move_failed = True
+                        logger.warning(f"Failed to move {src_item} to {final_dir}: {ex}")
+                if move_failed:
+                    # Removing the staging directory now would delete the very results that
+                    # could not be moved out of it. Leave it and tell the user where it is.
+                    logger.warning(f"Some results could not be moved and are left in {output_path}.")
+                else:
+                    remove_tree(output_path)
                 if final_reports:
-                    final_reports = [report.replace(output_path, final_dir) for report in final_reports]
+                    # Only rewrite the paths of the reports that really arrived in final_dir,
+                    # so a report left behind is reported where it actually is.
+                    moved_reports = []
+                    for report in final_reports:
+                        moved = report.replace(output_path, final_dir)
+                        moved_reports.append(moved if os.path.exists(moved) else report)
+                    final_reports = moved_reports
                     logger.info(f'Output File: {", ".join(final_reports)}')
         except Exception as ex:
             logger.debug(f"Error to remove temp files:{ex}")