[imp] add sparse checkout

README.rst

@@ -206,6 +206,27 @@ A real life example: applying a patch
         shell_command_after:
             - git am "$(git format-patch -1 XXXXXX -o ../patches)"
 
+Sparse Checkout
+---------------
+
+Git provides sparse-checkout to only checkout a set of files/directories which is
+very useful for more granular control over what we should keep. Especially useful
+when repository is getting big, or when you want to automatically install only
+specific modules from the directory.
+
+Looking at the example below, only ``product_brand`` will be checkout from remote.
+
+.. code-block:: yaml
+
+    ./product_attribute:
+        remotes:
+            oca: https://github.com/OCA/product-attribute.git
+        merges:
+            - oca 8.0
+        target: oca 8.0
+        sparse-checkout:
+            - product_brand
+
 Command line Usage
 ==================
 

git_aggregator/config.py

@@ -126,6 +126,15 @@ def get_repos(config, force=False):
                     cmds = [cmds]
                 commands = cmds
         repo_dict['shell_command_after'] = commands
+        # Handle sparse-checkout configuration
+        sparse_checkout = repo_data.get('sparse-checkout', None)
+        if sparse_checkout:
+            if isinstance(sparse_checkout, string_types):
+                sparse_checkout = [sparse_checkout]
+            elif not isinstance(sparse_checkout, list):
+                raise ConfigException(
+                    '%s: sparse-checkout must be a string or list of strings.' % directory)
+        repo_dict['sparse_checkout'] = sparse_checkout
         repo_list.append(repo_dict)
     return repo_list
 

git_aggregator/repo.py

@@ -36,7 +36,7 @@ class Repo:
 
     def __init__(self, cwd, remotes, merges, target,
                  shell_command_after=None, fetch_all=False, defaults=None,
-                 force=False):
+                 force=False, sparse_checkout=None):
         """Initialize a git repository aggregator
 
         :param cwd: path to the directory where to initialize the repository
@@ -55,6 +55,7 @@ def __init__(self, cwd, remotes, merges, target,
             Collection of default parameters to be passed to git.
         :param bool force:
             When ``False``, it will stop if repo is dirty.
+        :param sparse_checkout: list of paths to include in sparse-checkout
         """
         self.cwd = cwd
         self.remotes = remotes
@@ -67,6 +68,7 @@ def __init__(self, cwd, remotes, merges, target,
         self.shell_command_after = shell_command_after or []
         self.defaults = defaults or dict()
         self.force = force
+        self.sparse_checkout = sparse_checkout
 
     @property
     def git_version(self):
@@ -226,6 +228,9 @@ def init_repository(self, target_dir):
             # Speeds up cloning by functioning without a complete copy of
             # repository
             cmd += ('--filter=blob:none',)
+        # Enable sparse-checkout if configured
+        if self.sparse_checkout:
+            cmd += ('--no-checkout',)
         # Try to clone target branch, if it exists
         rtype, _sha = self.query_remote_ref(repository, branch)
         if rtype in {'branch', 'tag'}:
@@ -234,6 +239,17 @@ def init_repository(self, target_dir):
         cmd += self._fetch_options({})
         cmd += (repository, target_dir)
         self.log_call(cmd)
+
+        # Configure and apply sparse-checkout if specified
+        if self.sparse_checkout:
+            logger.info('Configuring sparse-checkout for %s', self.sparse_checkout)
+            # Enable sparse-checkout
+            self.log_call(['git', 'sparse-checkout', 'init', '--cone'], cwd=target_dir)
+            # Set the paths to include
+            self.log_call(['git', 'sparse-checkout', 'set'] + self.sparse_checkout, cwd=target_dir)
+            # Checkout the files
+            self.log_call(['git', 'checkout'], cwd=target_dir)
+
         return True
 
     def fetch(self):

tests/test_config.py

@@ -435,3 +435,55 @@ def test_fetch_all_true(self):
         config_yaml = dedent(config_yaml)
         repos = config.get_repos(self._parse_config(config_yaml))
         self.assertIs(repos[0]["fetch_all"], True)
+
+    def test_sparse_checkout_string(self):
+        """Test sparse-checkout with a single string path."""
+        config_yaml = """
+            ./test:
+                remotes:
+                    oca: https://github.com/test/test.git
+                merges:
+                    - oca 8.0
+                target: oca aggregated_branch_name
+                sparse-checkout: src/module1
+            """
+        config_yaml = dedent(config_yaml)
+        repos = config.get_repos(self._parse_config(config_yaml))
+        self.assertEqual(repos[0]["sparse_checkout"], ["src/module1"])
+
+    def test_sparse_checkout_list(self):
+        """Test sparse-checkout with a list of paths."""
+        config_yaml = """
+            ./test:
+                remotes:
+                    oca: https://github.com/test/test.git
+                merges:
+                    - oca 8.0
+                target: oca aggregated_branch_name
+                sparse-checkout:
+                    - src/module1
+                    - src/module2
+                    - docs
+            """
+        config_yaml = dedent(config_yaml)
+        repos = config.get_repos(self._parse_config(config_yaml))
+        self.assertEqual(repos[0]["sparse_checkout"], ["src/module1", "src/module2", "docs"])
+
+    def test_sparse_checkout_invalid_type(self):
+        """Test sparse-checkout with invalid type raises ConfigException."""
+        config_yaml = """
+            /test:
+                remotes:
+                    oca: https://github.com/test/test.git
+                merges:
+                    - oca 8.0
+                target: oca aggregated_branch_name
+                sparse-checkout: 123
+            """
+        config_yaml = dedent(config_yaml)
+        with self.assertRaises(ConfigException) as ex:
+            config.get_repos(self._parse_config(config_yaml))
+        self.assertEqual(
+            ex.exception.args[0],
+            '/test: sparse-checkout must be a string or list of strings.'
+        )

tests/test_repo.py

@@ -444,3 +444,87 @@ def test_multithreading(self):
 
         self.assertTrue(os.path.isfile(os.path.join(repo3_dir, 'tracked')))
         self.assertTrue(os.path.isfile(os.path.join(repo3_dir, 'tracked2')))
+
+    def test_sparse_checkout_single_path(self):
+        """Test sparse-checkout with a single path."""
+        # Create a directory structure in remote1
+        with WorkingDirectoryKeeper():
+            os.chdir(self.remote1)
+            os.makedirs('src/module1', exist_ok=True)
+            os.makedirs('src/module2', exist_ok=True)
+            git_write_commit(self.remote1, 'src/module1/file1.txt',
+                           'content1', msg='add module1 file')
+            git_write_commit(self.remote1, 'src/module2/file2.txt',
+                           'content2', msg='add module2 file')
+
+        remotes = [{
+            'name': 'r1',
+            'url': self.url_remote1
+        }]
+        merges = [{
+            'remote': 'r1',
+            'ref': 'main'
+        }]
+        target = {
+            'remote': 'r1',
+            'branch': 'agg'
+        }
+
+        # Test with sparse-checkout for only module1
+        repo = Repo(self.cwd, remotes, merges, target,
+                   sparse_checkout=['src/module1'])
+        repo.aggregate()
+
+        # module1 should be present
+        self.assertTrue(os.path.isfile(
+            os.path.join(self.cwd, 'src/module1/file1.txt')))
+        # module2 should not be checked out
+        self.assertFalse(os.path.exists(
+            os.path.join(self.cwd, 'src/module2')))
+
+    def test_sparse_checkout_multiple_paths(self):
+        """Test sparse-checkout with multiple paths."""
+        # Create a directory structure in remote1
+        with WorkingDirectoryKeeper():
+            os.chdir(self.remote1)
+            os.makedirs('docs', exist_ok=True)
+            os.makedirs('src/core', exist_ok=True)
+            os.makedirs('src/utils', exist_ok=True)
+            os.makedirs('tests', exist_ok=True)
+            git_write_commit(self.remote1, 'docs/readme.md',
+                           'docs content', msg='add docs')
+            git_write_commit(self.remote1, 'src/core/main.py',
+                           'core code', msg='add core')
+            git_write_commit(self.remote1, 'src/utils/helpers.py',
+                           'utils code', msg='add utils')
+            git_write_commit(self.remote1, 'tests/test_main.py',
+                           'test code', msg='add tests')
+
+        remotes = [{
+            'name': 'r1',
+            'url': self.url_remote1
+        }]
+        merges = [{
+            'remote': 'r1',
+            'ref': 'main'
+        }]
+        target = {
+            'remote': 'r1',
+            'branch': 'agg'
+        }
+
+        # Test with sparse-checkout for docs and src/core only
+        repo = Repo(self.cwd, remotes, merges, target,
+                   sparse_checkout=['docs', 'src/core'])
+        repo.aggregate()
+
+        # docs and src/core should be present
+        self.assertTrue(os.path.isfile(
+            os.path.join(self.cwd, 'docs/readme.md')))
+        self.assertTrue(os.path.isfile(
+            os.path.join(self.cwd, 'src/core/main.py')))
+        # src/utils and tests should not be checked out
+        self.assertFalse(os.path.exists(
+            os.path.join(self.cwd, 'src/utils')))
+        self.assertFalse(os.path.exists(
+            os.path.join(self.cwd, 'tests')))