From c378138af0e17d80da2d95dc8cf8600d96a0711c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Casta=C3=B1o=20Arteaga?= Date: Mon, 5 Oct 2026 16:27:28 +0200 Subject: [PATCH] Skip ignored packages in tracker sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergio Castaño Arteaga --- docs/metadata/artifacthub-repo.yml | 30 +++- internal/hub/repo.go | 23 +++ internal/hub/repo_test.go | 129 +++++++++++++++ internal/hub/tracker.go | 5 +- internal/repo/errors.go | 16 +- internal/repo/errors_test.go | 52 ++++++ internal/tracker/helpers.go | 33 ---- internal/tracker/helpers_test.go | 107 ------------ .../tracker/source/container/container.go | 9 +- .../source/container/container_test.go | 116 +++++++++++++ internal/tracker/source/falco/falco.go | 9 ++ internal/tracker/source/falco/falco_test.go | 58 +++++++ internal/tracker/source/generic/generic.go | 8 +- .../tracker/source/generic/generic_test.go | 55 +++++++ internal/tracker/source/helm/helm.go | 12 +- internal/tracker/source/helm/helm_test.go | 130 +++++++++++++++ .../tracker/source/helmplugin/helmplugin.go | 6 + .../source/helmplugin/helmplugin_test.go | 58 +++++++ internal/tracker/source/krew/krew.go | 6 + internal/tracker/source/krew/krew_test.go | 58 +++++++ internal/tracker/source/olm/olm.go | 6 + internal/tracker/source/olm/olm_test.go | 93 +++++++++++ internal/tracker/source/tekton/tekton.go | 14 +- internal/tracker/source/tekton/tekton_test.go | 152 ++++++++++++++++++ internal/tracker/tracker.go | 18 ++- internal/tracker/tracker_test.go | 87 ++++++++++ 26 files changed, 1131 insertions(+), 159 deletions(-) create mode 100644 internal/hub/repo_test.go create mode 100644 internal/tracker/source/container/container_test.go diff --git a/docs/metadata/artifacthub-repo.yml b/docs/metadata/artifacthub-repo.yml index f7f9a33897..d24a57dae0 100644 --- a/docs/metadata/artifacthub-repo.yml +++ b/docs/metadata/artifacthub-repo.yml @@ -8,14 +8,40 @@ # changed if the `index.yaml` file changes. For git based repositories, it does # when the hash of the last commit in the branch you set up changes. This does # NOT apply to ownership claim operations, which are processed immediately. -# -repositoryID: The ID of the Artifact Hub repository where the packages will be published to (optional, but it enables verified publisher) + +# ID of the Artifact Hub repository where the packages will be published to. +# It is displayed on the repository's card (ID) in the repositories tab of the +# Artifact Hub control panel. The next time the repository is processed, the +# repository will be marked as a verified publisher if the ID matches. +repositoryID: 7a5ac6a4-1a1c-4d2e-9b8f-3f5e2c1d0b9a # (optional, but it enables verified publisher) + +# Repository owners, who can claim the ownership of the repository from the +# Artifact Hub control panel. The user requesting the ownership claim must be +# listed here, and the email must match the one used to sign in to Artifact +# Hub. The repositoryID field can be omitted when claiming the ownership, as +# it isn't known yet. owners: # (optional, used to claim repository ownership) - name: user1 email: user1@email.com - name: user2 email: user2@email.com + +# Ignored packages are not indexed and Artifact Hub skips as much of their +# processing as possible, so most errors related to them won't be reported. +# Some errors may still be reported for them, such as those found while +# reading their metadata files or preparing their content. Ignored packages +# already indexed will be removed. +# +# The version is a regular expression matched against the package version as +# displayed in Artifact Hub (e.g. Helm chart versions are normalized, so `v1.0` +# becomes `1.0.0`). It is not anchored, so `1.0.0` also matches `11.0.0` or +# `1.0.0-rc.1`: use `^` and `$` to ignore specific versions only. A package +# can be listed multiple times. For Helm charts and container images, errors +# found while preparing ignored versions are not reported, so this can be used +# to stop errors in old versions that can't be fixed from being reported. ignore: # (optional, packages that should not be indexed by Artifact Hub) - name: package1 - name: package2 # Exact match version: beta # Regular expression (when omitted, all versions are ignored) + - name: package3 + version: ^(1\.0\.0|1\.0\.1)$ # Only versions 1.0.0 and 1.0.1 are ignored diff --git a/internal/hub/repo.go b/internal/hub/repo.go index 90e555e6a0..a2993ddbe5 100644 --- a/internal/hub/repo.go +++ b/internal/hub/repo.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "regexp" helmrepo "helm.sh/helm/v3/pkg/repo" ) @@ -355,6 +356,28 @@ type RepositoryMetadata struct { Ignore []*RepositoryIgnoreEntry `yaml:"ignore,omitempty"` } +// IgnoresPackage checks if the package version provided must be ignored +// according to the ignore list in the repository metadata. It is safe to +// call it on a nil RepositoryMetadata. +func (md *RepositoryMetadata) IgnoresPackage(name, version string) bool { + if md == nil { + return false + } + for _, e := range md.Ignore { + if e == nil || e.Name != name { + continue + } + if version == "" { + return true + } + matched, err := regexp.MatchString(e.Version, version) + if err == nil && matched { + return true + } + } + return false +} + // RepositoryIgnoreEntry represents an entry in the ignore list. This list is // meant to be used as a way to exclude packages from being indexed by Artifact // Hub. The name corresponds to a package name, and it must be an exact match. diff --git a/internal/hub/repo_test.go b/internal/hub/repo_test.go new file mode 100644 index 0000000000..e6f74c4524 --- /dev/null +++ b/internal/hub/repo_test.go @@ -0,0 +1,129 @@ +package hub + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRepositoryMetadataIgnoresPackage(t *testing.T) { + t.Parallel() + + testCases := []struct { + desc string + md *RepositoryMetadata + name string + version string + expectedResult bool + }{ + { + "name only entry, empty version", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1"}}}, + "pkg1", + "", + true, + }, + { + "name only entry, any version", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1"}}}, + "pkg1", + "1.0.0", + true, + }, + { + "different name", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg2"}}}, + "pkg1", + "1.0.0", + false, + }, + { + "empty ignore list", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{}}, + "pkg1", + "1.0.0", + false, + }, + { + "nil metadata", + nil, + "pkg1", + "1.0.0", + false, + }, + { + "unanchored regex matches prerelease", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "beta"}}}, + "pkg1", + "1.0.0-beta1", + true, + }, + { + "unanchored regex does not match", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "beta"}}}, + "pkg1", + "1.0.0", + false, + }, + { + "same version", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "1.0.0"}}}, + "pkg1", + "1.0.0", + true, + }, + { + "different version", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "1.0.0"}}}, + "pkg1", + "1.0.1", + false, + }, + { + "only second entry matches", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{ + {Name: "pkg1", Version: "^2\\.0\\.0$"}, + {Name: "pkg1", Version: "^1\\.0\\.0$"}, + }}, + "pkg1", + "1.0.0", + true, + }, + { + "anchored regex matches", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "^(1\\.0\\.0|1\\.0\\.1)$"}}}, + "pkg1", + "1.0.1", + true, + }, + { + "anchored regex does not match a longer version", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "^1\\.0\\.0$"}}}, + "pkg1", + "11.0.0", + false, + }, + { + "invalid regex", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "["}}}, + "pkg1", + "1.0.0", + false, + }, + { + "nil entry", + &RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{nil}}, + "pkg1", + "1.0.0", + false, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + result := tc.md.IgnoresPackage(tc.name, tc.version) + assert.Equal(t, tc.expectedResult, result) + }) + } +} diff --git a/internal/hub/tracker.go b/internal/hub/tracker.go index 142de6a5c2..e2737717f8 100644 --- a/internal/hub/tracker.go +++ b/internal/hub/tracker.go @@ -45,10 +45,13 @@ type TrackerSource interface { } // TrackerSourceInput represents the input provided to a TrackerSource to get -// the packages available in a repository when tracking it. +// the packages available in a repository when tracking it. Sources should skip +// the package versions ignored by the repository metadata provided, and must +// treat that metadata as read-only. type TrackerSourceInput struct { Repository *Repository RepositoryDigest string + RepositoryMetadata *RepositoryMetadata PackagesRegistered map[string]string BasePath string Svc *TrackerSourceServices diff --git a/internal/repo/errors.go b/internal/repo/errors.go index 80a5498019..f8681c860a 100644 --- a/internal/repo/errors.go +++ b/internal/repo/errors.go @@ -11,8 +11,10 @@ import ( ) const ( - // maxErrorsPerRepository represents the maximum number of errors we want - // to collect for a given repository. + // maxErrorsPerRepository represents the maximum number of errors stored + // for a given repository. It is applied after sorting the errors, so the + // stored text only depends on the errors produced, not on the order they + // were collected in. maxErrorsPerRepository = 100 ) @@ -52,9 +54,7 @@ func (c *ErrorsCollector) Append(repositoryID string, err string) { c.mu.Lock() defer c.mu.Unlock() - if len(c.errors[repositoryID]) < maxErrorsPerRepository { - c.errors[repositoryID] = append(c.errors[repositoryID], err) - } + c.errors[repositoryID] = append(c.errors[repositoryID], err) } // Flush aggregates all errors collected per repository as a single text and @@ -68,8 +68,12 @@ func (c *ErrorsCollector) Flush() { // a repository concurrently, and the order the errors are produced is // not guaranteed. In order to be able to notify users when something // goes wrong during repositories tracking or scanning, we need to be - // able to compare the errors produced among executions. + // able to compare the errors produced among executions. For the same + // reason, errors are truncated only after sorting them. sort.Strings(errors) + if len(errors) > maxErrorsPerRepository { + errors = errors[:maxErrorsPerRepository] + } var allErrors strings.Builder for i, err := range errors { diff --git a/internal/repo/errors_test.go b/internal/repo/errors_test.go index 91ba3c3131..0c0238ba7b 100644 --- a/internal/repo/errors_test.go +++ b/internal/repo/errors_test.go @@ -2,6 +2,8 @@ package repo import ( "context" + "fmt" + "strings" "testing" ) @@ -44,3 +46,53 @@ func TestCollector(t *testing.T) { }) } } + +func TestCollectorTruncation(t *testing.T) { + // Expected text: the first maxErrorsPerRepository errors once sorted + expectedErrors := make([]string, 0, maxErrorsPerRepository) + for i := range maxErrorsPerRepository { + expectedErrors = append(expectedErrors, fmt.Sprintf("error%03d", i)) + } + expectedText := strings.Join(expectedErrors, "\n") + + kinds := []struct { + kind ErrorsCollectorKind + expectedCall string + }{ + {Scanner, "SetLastScanningResults"}, + {Tracker, "SetLastTrackingResults"}, + } + orders := []struct { + name string + descending bool + }{ + {"ascending", false}, + {"descending", true}, + } + for _, k := range kinds { + for _, o := range orders { + t.Run(fmt.Sprintf("%s %s", k.expectedCall, o.name), func(t *testing.T) { + t.Parallel() + + // Setup errors collector + rm := &ManagerMock{} + ec := NewErrorsCollector(rm, k.kind) + + // Append more errors than the maximum allowed + const numErrors = 150 + for i := range numErrors { + n := i + if o.descending { + n = numErrors - 1 - i + } + ec.Append("repo1", fmt.Sprintf("error%03d", n)) + } + + // Flush errors and check only the first ones were stored + rm.On(k.expectedCall, context.Background(), "repo1", expectedText).Return(nil) + ec.Flush() + rm.AssertExpectations(t) + }) + } + } +} diff --git a/internal/tracker/helpers.go b/internal/tracker/helpers.go index 56c33c8882..cba848ae3d 100644 --- a/internal/tracker/helpers.go +++ b/internal/tracker/helpers.go @@ -3,7 +3,6 @@ package tracker import ( "context" "fmt" - "regexp" "github.com/artifacthub/hub/internal/hub" "github.com/artifacthub/hub/internal/tracker/source/container" @@ -161,35 +160,3 @@ func setVerifiedPublisherFlag( } return nil } - -// shouldIgnorePackage checks if the package provided should be ignored. -func shouldIgnorePackage(md *hub.RepositoryMetadata, name, version string) bool { - if md == nil { - return false - } - for _, ignoreEntry := range md.Ignore { - if matchesEntry(ignoreEntry, name, version) { - return true - } - } - return false -} - -// matchesEntry checks if the package name and version provide match a given -// ignore entry. -func matchesEntry(ignoreEntry *hub.RepositoryIgnoreEntry, name, version string) bool { - if ignoreEntry.Name != name { - return false - } - if version == "" { - return true - } - versionMatch, err := regexp.Match(ignoreEntry.Version, []byte(version)) - if err != nil { - return false - } - if versionMatch { - return true - } - return false -} diff --git a/internal/tracker/helpers_test.go b/internal/tracker/helpers_test.go index f5ef50b079..acf1100ae6 100644 --- a/internal/tracker/helpers_test.go +++ b/internal/tracker/helpers_test.go @@ -343,110 +343,3 @@ func TestSetVerifiedPublisherFlag(t *testing.T) { rm.AssertExpectations(t) }) } - -func TestShouldIgnorePackage(t *testing.T) { - testCases := []struct { - md *hub.RepositoryMetadata - name string - version string - expectedResult bool - }{ - { - &hub.RepositoryMetadata{ - Ignore: []*hub.RepositoryIgnoreEntry{ - { - Name: "pkg1", - }, - }, - }, - "pkg1", - "", - true, - }, - { - &hub.RepositoryMetadata{ - Ignore: []*hub.RepositoryIgnoreEntry{ - { - Name: "pkg1", - }, - }, - }, - "pkg1", - "1.0.0", - true, - }, - { - &hub.RepositoryMetadata{ - Ignore: []*hub.RepositoryIgnoreEntry{ - { - Name: "pkg2", - }, - }, - }, - "pkg1", - "1.0.0", - false, - }, - { - &hub.RepositoryMetadata{ - Ignore: []*hub.RepositoryIgnoreEntry{}, - }, - "pkg1", - "1.0.0", - false, - }, - { - nil, - "pkg1", - "1.0.0", - false, - }, - { - &hub.RepositoryMetadata{ - Ignore: []*hub.RepositoryIgnoreEntry{ - { - Name: "pkg1", - Version: "beta", - }, - }, - }, - "pkg1", - "1.0.0-beta1", - true, - }, - { - &hub.RepositoryMetadata{ - Ignore: []*hub.RepositoryIgnoreEntry{ - { - Name: "pkg1", - Version: "1.0.0", - }, - }, - }, - "pkg1", - "1.0.0", - true, - }, - { - &hub.RepositoryMetadata{ - Ignore: []*hub.RepositoryIgnoreEntry{ - { - Name: "pkg1", - Version: "1.0.0", - }, - }, - }, - "pkg1", - "1.0.1", - false, - }, - } - for i, tc := range testCases { - t.Run(fmt.Sprintf("Test case %d", i), func(t *testing.T) { - t.Parallel() - - result := shouldIgnorePackage(tc.md, tc.name, tc.version) - assert.Equal(t, tc.expectedResult, result) - }) - } -} diff --git a/internal/tracker/source/container/container.go b/internal/tracker/source/container/container.go index f7871e734c..d4349dab2e 100644 --- a/internal/tracker/source/container/container.go +++ b/internal/tracker/source/container/container.go @@ -114,6 +114,12 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) Name: path.Base(s.i.Repository.URL), Version: tag.Name, } + + // Skip package versions ignored in the repository metadata + if s.i.RepositoryMetadata.IgnoresPackage(p.Name, p.Version) { + continue + } + key := pkg.BuildKey(p) if _, ok := s.i.PackagesRegistered[key]; !ok || tag.Mutable { tagsToProcess = append(tagsToProcess, tag.Name) @@ -153,8 +159,9 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) s.warn(fmt.Errorf("error preparing package (tag: %s): %w", tag, err)) return } + key := pkg.BuildKey(p) mu.Lock() - packagesAvailable[pkg.BuildKey(p)] = p + packagesAvailable[key] = p mu.Unlock() }(tag) } diff --git a/internal/tracker/source/container/container_test.go b/internal/tracker/source/container/container_test.go new file mode 100644 index 0000000000..ba96fa6ddc --- /dev/null +++ b/internal/tracker/source/container/container_test.go @@ -0,0 +1,116 @@ +package container + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/artifacthub/hub/internal/hub" + "github.com/artifacthub/hub/internal/tracker/source" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestTrackerSourceIgnoredPackages(t *testing.T) { + t.Parallel() + + testCases := []struct { + desc string + tags []hub.ContainerImageTag + expectRegistryRequests bool + }{ + { + "ignored tags skipped, tag not ignored fetched", + []hub.ContainerImageTag{ + {Name: "1.0.0", Mutable: true}, + {Name: "2.0.0", Mutable: false}, + {Name: "3.0.0", Mutable: false}, + {Name: "v4.0.0", Mutable: true}, + }, + true, + }, + { + "all tags to process ignored", + []hub.ContainerImageTag{ + {Name: "1.0.0", Mutable: true}, + {Name: "2.0.0", Mutable: false}, + {Name: "3.0.0", Mutable: false}, + }, + false, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + // Setup fake registry + var requests atomic.Int64 + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer registry.Close() + host := strings.TrimPrefix(registry.URL, "http://") + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + sw.Ec.Test(t) + sw.Hc.Test(t) + sw.Op.Test(t) + sw.Is.Test(t) + sw.Sc.Test(t) + var logs bytes.Buffer + sw.Svc.Logger = zerolog.New(zerolog.SyncWriter(&logs)) + data, err := json.Marshal(hub.ContainerImageData{Tags: tc.tags}) + require.NoError(t, err) + i := &hub.TrackerSourceInput{ + Repository: &hub.Repository{ + RepositoryID: "repo1", + URL: hub.RepositoryOCIPrefix + host + "/ns/img", + Data: data, + }, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{ + { + Name: "img", + // 4 matches a normalized 4.0.0, but not the raw v4.0.0 tag + Version: `^[124]\.0\.0$`, + }, + }, + }, + PackagesRegistered: map[string]string{ + "img@2.0.0": "digest2", + "img@3.0.0": "digest3", + }, + Svc: sw.Svc, + } + if tc.expectRegistryRequests { + sw.Ec.On("Append", "repo1", mock.Anything).Return().Once() + } + + // Run test and check expectations + packages, err := NewTrackerSource(i).GetPackagesAvailable() + require.NoError(t, err) + assert.Equal(t, map[string]*hub.Package{ + "img@3.0.0": { + Name: "img", + Version: "3.0.0", + Digest: hub.HasNotChanged, + }, + }, packages) + if tc.expectRegistryRequests { + assert.Positive(t, requests.Load()) + } else { + assert.Zero(t, requests.Load()) + } + assert.NotContains(t, logs.String(), "recover") + sw.AssertExpectations(t) + }) + } +} diff --git a/internal/tracker/source/falco/falco.go b/internal/tracker/source/falco/falco.go index dcd71dbf48..8e6c199947 100644 --- a/internal/tracker/source/falco/falco.go +++ b/internal/tracker/source/falco/falco.go @@ -81,6 +81,9 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) s.warn(fmt.Errorf("error preparing package: %w", err)) return nil } + if p == nil { + return nil + } packagesAvailable[pkg.BuildKey(p)] = p return nil @@ -93,6 +96,7 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) } // preparePackage prepares a package version using the rules metadata provided. +// It returns nil when the package version must be ignored. func (s *TrackerSource) preparePackage(r *hub.Repository, md *RulesMetadata, pkgPath string) (*hub.Package, error) { // Parse and validate version sv, err := semver.NewVersion(md.Version) @@ -101,6 +105,11 @@ func (s *TrackerSource) preparePackage(r *hub.Repository, md *RulesMetadata, pkg } version := sv.String() + // Skip package versions ignored in the repository metadata + if s.i.RepositoryMetadata.IgnoresPackage(md.Name, version) { + return nil, nil + } + // Prepare package from metadata p := &hub.Package{ Name: md.Name, diff --git a/internal/tracker/source/falco/falco_test.go b/internal/tracker/source/falco/falco_test.go index 0dd6c5d445..de1244a736 100644 --- a/internal/tracker/source/falco/falco_test.go +++ b/internal/tracker/source/falco/falco_test.go @@ -146,4 +146,62 @@ func TestTrackerSource(t *testing.T) { assert.NoError(t, err) sw.AssertExpectations(t) }) + + t.Run("ignored package not returned, logo not processed", func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + i := &hub.TrackerSourceInput{ + Repository: &hub.Repository{ + URL: "https://github.com/org1/repo1/path/to/packages", + Branch: "main", + }, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{{Name: "test", Version: `^0\.1\.0$`}}, + }, + BasePath: "testdata/path4", + Svc: sw.Svc, + } + + // Run test and check expectations + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.Equal(t, map[string]*hub.Package{}, packages) + assert.NoError(t, err) + sw.AssertExpectations(t) + }) + + t.Run("package returned when ignore entry version does not match", func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + i := &hub.TrackerSourceInput{ + Repository: &hub.Repository{ + URL: "https://github.com/org1/repo1/path/to/packages", + Branch: "main", + }, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{{Name: "test", Version: `^0\.2\.0$`}}, + }, + BasePath: "testdata/path4", + Svc: sw.Svc, + } + sw.Is.On("DownloadAndSaveImage", sw.Svc.Ctx, logoImageURL).Return("logoImageID", nil) + + // Run test and check expectations + p := source.ClonePackage(basePkg) + p.Repository = i.Repository + p.LogoURL = logoImageURL + p.LogoImageID = "logoImageID" + p.Data = map[string]interface{}{ + rulesKey: []*Rule{{Raw: "Falco rules in YAML"}}, + } + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.Equal(t, map[string]*hub.Package{ + pkg.BuildKey(p): p, + }, packages) + assert.NoError(t, err) + sw.AssertExpectations(t) + }) } diff --git a/internal/tracker/source/generic/generic.go b/internal/tracker/source/generic/generic.go index 3ce75cfec0..7eaef7bbf0 100644 --- a/internal/tracker/source/generic/generic.go +++ b/internal/tracker/source/generic/generic.go @@ -94,7 +94,7 @@ type TrackerSource struct { } // NewTrackerSource creates a new TrackerSource instance. -func NewTrackerSource(i *hub.TrackerSourceInput, opts ...func(s *TrackerSource)) *TrackerSource { +func NewTrackerSource(i *hub.TrackerSourceInput) *TrackerSource { return &TrackerSource{i: i} } @@ -135,6 +135,12 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) s.warn(err) return nil } + + // Skip package versions ignored in the repository metadata + if s.i.RepositoryMetadata.IgnoresPackage(p.Name, p.Version) { + return nil + } + p.RelativePath = strings.TrimPrefix(pkgPath, s.i.BasePath) packagesAvailable[pkg.BuildKey(p)] = p diff --git a/internal/tracker/source/generic/generic_test.go b/internal/tracker/source/generic/generic_test.go index 1329944660..afc24c959a 100644 --- a/internal/tracker/source/generic/generic_test.go +++ b/internal/tracker/source/generic/generic_test.go @@ -337,4 +337,59 @@ func TestTrackerSource(t *testing.T) { assert.NoError(t, err) sw.AssertExpectations(t) }) + + t.Run("ignored package not returned, logo not processed", func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + i := &hub.TrackerSourceInput{ + Repository: &hub.Repository{ + Kind: hub.TBAction, + }, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{{Name: "pkg1", Version: `^1\.0\.0$`}}, + }, + BasePath: "testdata/path4", + Svc: sw.Svc, + } + + // Run test and check expectations + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.Equal(t, map[string]*hub.Package{}, packages) + assert.NoError(t, err) + sw.AssertExpectations(t) + }) + + t.Run("package returned when ignore entry version does not match", func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + i := &hub.TrackerSourceInput{ + Repository: &hub.Repository{ + Kind: hub.OPA, + }, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{{Name: "pkg1", Version: `^2\.0\.0$`}}, + }, + BasePath: "testdata/path6", + Svc: sw.Svc, + } + sw.Is.On("SaveImage", sw.Svc.Ctx, imageData).Return("logoImageID", nil) + + // Run test and check expectations + p := source.ClonePackage(basePkg) + p.Repository = i.Repository + p.LogoImageID = "logoImageID" + p.Data[OPAPoliciesKey] = map[string]string{ + "policy1.rego": "policy content\n", + } + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.Equal(t, map[string]*hub.Package{ + pkg.BuildKey(p): p, + }, packages) + assert.NoError(t, err) + sw.AssertExpectations(t) + }) } diff --git a/internal/tracker/source/helm/helm.go b/internal/tracker/source/helm/helm.go index 70d5f433c6..e05df9b1c0 100644 --- a/internal/tracker/source/helm/helm.go +++ b/internal/tracker/source/helm/helm.go @@ -157,8 +157,12 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) s.warn(chartVersion.Metadata, fmt.Errorf("error preparing package: %w", err)) return } + if p == nil { + return + } + key := pkg.BuildKey(p) mu.Lock() - packagesAvailable[pkg.BuildKey(p)] = p + packagesAvailable[key] = p mu.Unlock() }(chartVersion) } @@ -220,6 +224,7 @@ func (s *TrackerSource) getCharts() (map[string][]*helmrepo.ChartVersion, error) } // preparePackage prepares a package version using the chart version provided. +// It returns nil when the package version must be ignored. func (s *TrackerSource) preparePackage(chartVersion *helmrepo.ChartVersion) (*hub.Package, error) { // Parse package version md := chartVersion.Metadata @@ -229,6 +234,11 @@ func (s *TrackerSource) preparePackage(chartVersion *helmrepo.ChartVersion) (*hu } version := sv.String() + // Skip package versions ignored in the repository metadata + if s.i.RepositoryMetadata.IgnoresPackage(chartVersion.Name, version) { + return nil, nil + } + // Prepare chart archive url if len(chartVersion.URLs) == 0 { return nil, errors.New("chart version does not contain any url") diff --git a/internal/tracker/source/helm/helm_test.go b/internal/tracker/source/helm/helm_test.go index 78ed3f09b3..dc62015d46 100644 --- a/internal/tracker/source/helm/helm_test.go +++ b/internal/tracker/source/helm/helm_test.go @@ -1,6 +1,7 @@ package helm import ( + "bytes" "errors" "io" "net/http" @@ -17,6 +18,7 @@ import ( "github.com/artifacthub/hub/internal/tests" "github.com/artifacthub/hub/internal/tracker/source" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -539,6 +541,134 @@ func TestTrackerSource(t *testing.T) { tg.AssertExpectations(t) sw.AssertExpectations(t) }) + +} + +func TestTrackerSourceIgnoredPackages(t *testing.T) { + t.Parallel() + + chartVersion := func(version, digest string) *helmrepo.ChartVersion { + return &helmrepo.ChartVersion{ + Metadata: &chart.Metadata{ + APIVersion: "v2", + Name: "pkg1", + Version: version, + Icon: "http://icon.url", + }, + URLs: []string{"https://repo.url/pkg1-" + version + ".tgz"}, + Digest: digest, + } + } + + testCases := []struct { + desc string + repositoryURL string + ignore []*hub.RepositoryIgnoreEntry + packagesRegistered map[string]string + chartVersions []*helmrepo.ChartVersion // Used in http repositories + tags []string // Used in oci repositories + expectedPackages []*hub.Package + }{ + { + desc: "ignored package not prepared (http)", + repositoryURL: "https://repo.url", + ignore: []*hub.RepositoryIgnoreEntry{{Name: "pkg1"}}, + chartVersions: []*helmrepo.ChartVersion{chartVersion("1.0.0", "")}, + }, + { + desc: "only versions not ignored returned (http)", + repositoryURL: "https://repo.url", + ignore: []*hub.RepositoryIgnoreEntry{{Name: "pkg1", Version: `^1\.0\.0$`}}, + packagesRegistered: map[string]string{"pkg1@1.1.0": "digest2"}, + chartVersions: []*helmrepo.ChartVersion{ + chartVersion("v1.0.0", "digest1"), + chartVersion("1.1.0", "digest2"), + }, + expectedPackages: []*hub.Package{ + { + Name: "pkg1", + Version: "1.1.0", + Digest: "digest2", + ContentURL: "https://repo.url/pkg1-1.1.0.tgz", + }, + }, + }, + { + desc: "ignored package registered with the same digest not returned (http)", + repositoryURL: "https://repo.url", + ignore: []*hub.RepositoryIgnoreEntry{{Name: "pkg1", Version: `^1\.0\.0$`}}, + packagesRegistered: map[string]string{"pkg1@1.0.0": "digest1"}, + chartVersions: []*helmrepo.ChartVersion{chartVersion("1.0.0", "digest1")}, + }, + { + desc: "only versions not ignored returned (oci)", + repositoryURL: "oci://registry/namespace/pkg1", + ignore: []*hub.RepositoryIgnoreEntry{{Name: "pkg1", Version: `^1\.0\.0$`}}, + packagesRegistered: map[string]string{"pkg1@1.1.0": ""}, + tags: []string{"1.0.0", "1.1.0"}, + expectedPackages: []*hub.Package{ + { + Name: "pkg1", + Version: "1.1.0", + ContentURL: "oci://registry/namespace/pkg1:1.1.0", + }, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + // Setup services and expectations. Unexpected calls to the mocks + // must fail the test, as a panic inside a worker would be + // recovered and the package silently dropped. + sw := source.NewTestsServicesWrapper() + sw.Ec.Test(t) + sw.Hc.Test(t) + sw.Op.Test(t) + sw.Is.Test(t) + sw.Sc.Test(t) + var logs bytes.Buffer + sw.Svc.Logger = zerolog.New(zerolog.SyncWriter(&logs)) + i := &hub.TrackerSourceInput{ + Repository: &hub.Repository{ + URL: tc.repositoryURL, + }, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: tc.ignore, + }, + PackagesRegistered: tc.packagesRegistered, + Svc: sw.Svc, + } + il := &repo.HelmIndexLoaderMock{} + il.Test(t) + tg := &oci.TagsGetterMock{} + tg.Test(t) + if tc.tags != nil { + tg.On("Tags", i.Svc.Ctx, i.Repository, true).Return(tc.tags, nil) + } else { + il.On("LoadIndex", i.Repository).Return(&helmrepo.IndexFile{ + Entries: map[string]helmrepo.ChartVersions{ + "pkg1": tc.chartVersions, + }, + }, "", nil) + } + + // Run test and check expectations + expectedPackages := make(map[string]*hub.Package) + for _, p := range tc.expectedPackages { + p.Repository = i.Repository + expectedPackages[pkg.BuildKey(p)] = p + } + packages, err := NewTrackerSource(i, withIndexLoader(il), withOCITagsGetter(tg)).GetPackagesAvailable() + assert.Equal(t, expectedPackages, packages) + assert.NoError(t, err) + assert.NotContains(t, logs.String(), "recover") + il.AssertExpectations(t) + tg.AssertExpectations(t) + sw.AssertExpectations(t) + }) + } } func TestExtractContainersImages(t *testing.T) { diff --git a/internal/tracker/source/helmplugin/helmplugin.go b/internal/tracker/source/helmplugin/helmplugin.go index 3f13fc6cfa..51379a9506 100644 --- a/internal/tracker/source/helmplugin/helmplugin.go +++ b/internal/tracker/source/helmplugin/helmplugin.go @@ -69,6 +69,12 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) s.warn(fmt.Errorf("error preparing package %s version %s: %w", md.Name, md.Version, err)) return nil } + + // Skip package versions ignored in the repository metadata + if s.i.RepositoryMetadata.IgnoresPackage(p.Name, p.Version) { + return nil + } + packagesAvailable[pkg.BuildKey(p)] = p return nil diff --git a/internal/tracker/source/helmplugin/helmplugin_test.go b/internal/tracker/source/helmplugin/helmplugin_test.go index 2140791442..378af6b200 100644 --- a/internal/tracker/source/helmplugin/helmplugin_test.go +++ b/internal/tracker/source/helmplugin/helmplugin_test.go @@ -7,6 +7,7 @@ import ( "github.com/artifacthub/hub/internal/pkg" "github.com/artifacthub/hub/internal/tracker/source" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestTrackerSource(t *testing.T) { @@ -103,3 +104,60 @@ func TestTrackerSource(t *testing.T) { sw.AssertExpectations(t) }) } + +func TestTrackerSourceIgnoredPackages(t *testing.T) { + t.Parallel() + + testCases := []struct { + desc string + ignoredVersion string + expectedReturned bool + }{ + { + "ignored package not returned", + `^0\.1\.0$`, + false, + }, + { + "package returned when ignore entry version does not match", + `^0\.2\.0$`, + true, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + r := &hub.Repository{} + i := &hub.TrackerSourceInput{ + Repository: r, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{{Name: "test-plugin", Version: tc.ignoredVersion}}, + }, + BasePath: "testdata/path4", + Svc: sw.Svc, + } + + // Run test and check expectations (when returned, the package must + // be the same as the one returned without an ignore list) + expectedPackages := map[string]*hub.Package{} + if tc.expectedReturned { + var err error + expectedPackages, err = NewTrackerSource(&hub.TrackerSourceInput{ + Repository: r, + BasePath: "testdata/path4", + Svc: sw.Svc, + }).GetPackagesAvailable() + require.NoError(t, err) + require.Len(t, expectedPackages, 1) + require.Contains(t, expectedPackages, "test-plugin@0.1.0") + } + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.Equal(t, expectedPackages, packages) + assert.NoError(t, err) + sw.AssertExpectations(t) + }) + } +} diff --git a/internal/tracker/source/krew/krew.go b/internal/tracker/source/krew/krew.go index b4b5281075..a74bf5c6fd 100644 --- a/internal/tracker/source/krew/krew.go +++ b/internal/tracker/source/krew/krew.go @@ -103,6 +103,12 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) )) continue } + + // Skip package versions ignored in the repository metadata + if s.i.RepositoryMetadata.IgnoresPackage(p.Name, p.Version) { + continue + } + packagesAvailable[pkg.BuildKey(p)] = p } diff --git a/internal/tracker/source/krew/krew_test.go b/internal/tracker/source/krew/krew_test.go index c4cec4d393..55254eb920 100644 --- a/internal/tracker/source/krew/krew_test.go +++ b/internal/tracker/source/krew/krew_test.go @@ -8,6 +8,7 @@ import ( "github.com/artifacthub/hub/internal/pkg" "github.com/artifacthub/hub/internal/tracker/source" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestTrackerSource(t *testing.T) { @@ -146,3 +147,60 @@ func TestTrackerSource(t *testing.T) { sw.AssertExpectations(t) }) } + +func TestTrackerSourceIgnoredPackages(t *testing.T) { + t.Parallel() + + testCases := []struct { + desc string + ignoredVersion string + expectedReturned bool + }{ + { + "ignored package not returned", + `^0\.1\.0$`, + false, + }, + { + "package returned when ignore entry version does not match", + `^0\.2\.0$`, + true, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + r := &hub.Repository{} + i := &hub.TrackerSourceInput{ + Repository: r, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{{Name: "test-plugin", Version: tc.ignoredVersion}}, + }, + BasePath: "testdata/path4", + Svc: sw.Svc, + } + + // Run test and check expectations (when returned, the package must + // be the same as the one returned without an ignore list) + expectedPackages := map[string]*hub.Package{} + if tc.expectedReturned { + var err error + expectedPackages, err = NewTrackerSource(&hub.TrackerSourceInput{ + Repository: r, + BasePath: "testdata/path4", + Svc: sw.Svc, + }).GetPackagesAvailable() + require.NoError(t, err) + require.Len(t, expectedPackages, 1) + require.Contains(t, expectedPackages, "test-plugin@0.1.0") + } + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.Equal(t, expectedPackages, packages) + assert.NoError(t, err) + sw.AssertExpectations(t) + }) + } +} diff --git a/internal/tracker/source/olm/olm.go b/internal/tracker/source/olm/olm.go index a7fca55a5a..2b626f0391 100644 --- a/internal/tracker/source/olm/olm.go +++ b/internal/tracker/source/olm/olm.go @@ -95,6 +95,12 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) s.warn(fmt.Errorf("error preparing package %s version %s: %w", md.Name, md.Version, err)) return nil } + + // Skip package versions ignored in the repository metadata + if s.i.RepositoryMetadata.IgnoresPackage(p.Name, p.Version) { + return nil + } + packagesAvailable[pkg.BuildKey(p)] = p logoImageID, err := s.prepareLogoImage(md) if err != nil { diff --git a/internal/tracker/source/olm/olm_test.go b/internal/tracker/source/olm/olm_test.go index 4dcdece934..b0d4c7c7c6 100644 --- a/internal/tracker/source/olm/olm_test.go +++ b/internal/tracker/source/olm/olm_test.go @@ -259,4 +259,97 @@ func TestTrackerSource(t *testing.T) { assert.NoError(t, err) sw.AssertExpectations(t) }) + + t.Run("ignored package not returned, logo not processed (package manifest format)", func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + i := &hub.TrackerSourceInput{ + Repository: &hub.Repository{}, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{{Name: "test-operator", Version: `^0\.1\.0$`}}, + }, + BasePath: "testdata/path4", + Svc: sw.Svc, + } + + // Run test and check expectations + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.Equal(t, map[string]*hub.Package{}, packages) + assert.NoError(t, err) + sw.AssertExpectations(t) + }) + + t.Run("package returned when ignore entry version does not match (package manifest format)", func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + i := &hub.TrackerSourceInput{ + Repository: &hub.Repository{}, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{{Name: "test-operator", Version: `^0\.2\.0$`}}, + }, + BasePath: "testdata/path4", + Svc: sw.Svc, + } + sw.Is.On("SaveImage", sw.Svc.Ctx, imageData).Return("logoImageID", nil) + + // Run test and check expectations + p := source.ClonePackage(basePkg) + p.Repository = i.Repository + p.LogoImageID = "logoImageID" + p.Digest = "8593896476590401712" + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.Equal(t, map[string]*hub.Package{ + pkg.BuildKey(p): p, + }, packages) + assert.NoError(t, err) + sw.AssertExpectations(t) + }) + + t.Run("ignored version not used to set channels (bundle format)", func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + i := &hub.TrackerSourceInput{ + Repository: &hub.Repository{}, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{{Name: "test-operator", Version: `^0\.2\.0$`}}, + }, + BasePath: "testdata/path5", + Svc: sw.Svc, + } + sw.Is.On("SaveImage", sw.Svc.Ctx, imageData).Return("logoImageID", nil).Once() + + // Run test and check expectations + p := source.ClonePackage(basePkg) + p.Repository = i.Repository + p.Category = hub.Security + p.LogoImageID = "logoImageID" + p.Channels = []*hub.Channel{ + { + Name: "alpha", + Version: "0.1.0", + }, + { + Name: "stable", + Version: "0.1.0", + }, + } + p.DefaultChannel = "stable" + p.Data = map[string]interface{}{ + formatKey: "bundle", + isGlobalOperatorKey: true, + } + p.Digest = "17802529201523149988" + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.Equal(t, map[string]*hub.Package{ + pkg.BuildKey(p): p, + }, packages) + assert.NoError(t, err) + sw.AssertExpectations(t) + }) } diff --git a/internal/tracker/source/tekton/tekton.go b/internal/tracker/source/tekton/tekton.go index 54775085cf..f76d61579b 100644 --- a/internal/tracker/source/tekton/tekton.go +++ b/internal/tracker/source/tekton/tekton.go @@ -161,7 +161,7 @@ func (s *TrackerSource) processDirBasedCatalog() (map[string]*hub.Package, error } for _, v := range versions { // If the path is not a directory or a ~valid semver version, we skip it - if !p.IsDir() { + if !v.IsDir() { continue } sv, err := semver.NewVersion(v.Name()) @@ -192,6 +192,12 @@ func (s *TrackerSource) processDirBasedCatalog() (map[string]*hub.Package, error s.warn(fmt.Errorf("error preparing package %s version %s: %w", pkgName, v.Name(), err)) continue } + + // Skip package versions ignored in the repository metadata + if s.i.RepositoryMetadata.IgnoresPackage(p.Name, p.Version) { + continue + } + packagesAvailable[pkg.BuildKey(p)] = p } } @@ -261,6 +267,12 @@ func (s *TrackerSource) processGitBasedCatalog() (map[string]*hub.Package, error s.warn(fmt.Errorf("error preparing package %s version %s: %w", pkgName, sv.String(), err)) continue } + + // Skip package versions ignored in the repository metadata + if s.i.RepositoryMetadata.IgnoresPackage(p.Name, p.Version) { + continue + } + packagesAvailable[pkg.BuildKey(p)] = p } diff --git a/internal/tracker/source/tekton/tekton_test.go b/internal/tracker/source/tekton/tekton_test.go index f5e27f6c45..7bce1d6beb 100644 --- a/internal/tracker/source/tekton/tekton_test.go +++ b/internal/tracker/source/tekton/tekton_test.go @@ -1,15 +1,21 @@ package tekton import ( + "bytes" "encoding/json" "fmt" "os" + "path/filepath" "testing" + "time" "github.com/artifacthub/hub/internal/hub" "github.com/artifacthub/hub/internal/pkg" "github.com/artifacthub/hub/internal/tracker/source" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestTrackerSource(t *testing.T) { @@ -365,4 +371,150 @@ func TestTrackerSource(t *testing.T) { assert.NoError(t, err) sw.AssertExpectations(t) }) + + t.Run("version entries that are not directories are skipped (dir based)", func(t *testing.T) { + t.Parallel() + + // Setup catalog with a version entry that is a regular file + basePath := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(basePath, "task1"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(basePath, "task1", "0.2.0"), []byte("not a dir"), 0o600)) + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + i := &hub.TrackerSourceInput{ + Repository: &hub.Repository{ + Kind: hub.TektonTask, + Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)), + }, + BasePath: basePath, + Svc: sw.Svc, + } + + // Run test and check expectations + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.Equal(t, map[string]*hub.Package{}, packages) + assert.NoError(t, err) + sw.AssertExpectations(t) + }) + + t.Run("ignored package not returned (git based)", func(t *testing.T) { + t.Parallel() + + // Setup git based catalog with two versions of a task + repoPath := t.TempDir() + basePath := filepath.Join(repoPath, "catalog") + gr, err := git.PlainInit(repoPath, false) + require.NoError(t, err) + wt, err := gr.Worktree() + require.NoError(t, err) + manifest, err := os.ReadFile("testdata/path3/task1/0.1/task1.yaml") + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(basePath, "task1"), 0o755)) + manifestPath := filepath.Join(basePath, "task1", "task1.yaml") + for _, version := range []string{"0.1.0", "0.2.0"} { + versionManifest := bytes.Replace( + manifest, + []byte(`app.kubernetes.io/version: "0.1.0"`), + []byte(`app.kubernetes.io/version: "`+version+`"`), + 1, + ) + require.Contains(t, string(versionManifest), `app.kubernetes.io/version: "`+version+`"`) + err := os.WriteFile(manifestPath, versionManifest, 0o600) // #nosec G703 -- test path is under t.TempDir() + require.NoError(t, err) + _, err = wt.Add("catalog/task1/task1.yaml") + require.NoError(t, err) + hash, err := wt.Commit("version "+version, &git.CommitOptions{ + Author: &object.Signature{Name: "test", Email: "test@email.com", When: time.Now()}, + }) + require.NoError(t, err) + _, err = gr.CreateTag("v"+version, hash, nil) + require.NoError(t, err) + } + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + i := &hub.TrackerSourceInput{ + Repository: &hub.Repository{ + Kind: hub.TektonTask, + URL: "https://github.com/user/repo/path", + Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonGitBasedVersioning)), + }, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{{Name: "task1", Version: `^0\.1\.0$`}}, + }, + BasePath: basePath, + Svc: sw.Svc, + } + + // Run test and check expectations + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.NoError(t, err) + require.Len(t, packages, 1) + require.Contains(t, packages, "task1@0.2.0") + assert.Equal(t, "task1", packages["task1@0.2.0"].Name) + assert.Equal(t, "0.2.0", packages["task1@0.2.0"].Version) + sw.AssertExpectations(t) + }) +} + +func TestTrackerSourceIgnoredPackages(t *testing.T) { + t.Parallel() + + testCases := []struct { + desc string + ignoredVersion string + expectedReturned bool + }{ + { + "ignored package not returned", + `^0\.1\.0$`, + false, + }, + { + "package returned when ignore entry version does not match", + `^0\.2\.0$`, + true, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + sw := source.NewTestsServicesWrapper() + r := &hub.Repository{ + Kind: hub.TektonTask, + URL: "https://github.com/user/repo/path", + Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)), + } + i := &hub.TrackerSourceInput{ + Repository: r, + RepositoryMetadata: &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{{Name: "task1", Version: tc.ignoredVersion}}, + }, + BasePath: "testdata/path3", + Svc: sw.Svc, + } + + // Run test and check expectations (when returned, the package must + // be the same as the one returned without an ignore list) + expectedPackages := map[string]*hub.Package{} + if tc.expectedReturned { + var err error + expectedPackages, err = NewTrackerSource(&hub.TrackerSourceInput{ + Repository: r, + BasePath: "testdata/path3", + Svc: sw.Svc, + }).GetPackagesAvailable() + require.NoError(t, err) + require.Len(t, expectedPackages, 1) + require.Contains(t, expectedPackages, "task1@0.1.0") + } + packages, err := NewTrackerSource(i).GetPackagesAvailable() + assert.Equal(t, expectedPackages, packages) + assert.NoError(t, err) + sw.AssertExpectations(t) + }) + } } diff --git a/internal/tracker/tracker.go b/internal/tracker/tracker.go index 85e1cc84da..42c1f06462 100644 --- a/internal/tracker/tracker.go +++ b/internal/tracker/tracker.go @@ -70,6 +70,7 @@ func (t *Tracker) Run() error { i := &hub.TrackerSourceInput{ Repository: t.r, RepositoryDigest: remoteDigest, + RepositoryMetadata: md, PackagesRegistered: packagesRegistered, BasePath: basePath, Svc: &hub.TrackerSourceServices{ @@ -104,8 +105,9 @@ func (t *Tracker) Run() error { continue } - // Check if this package should be ignored - if shouldIgnorePackage(md, p.Name, p.Version) { + // Check if this package should be ignored (safety net, sources are + // expected to have filtered ignored packages already) + if md.IgnoresPackage(p.Name, p.Version) { continue } @@ -124,8 +126,11 @@ func (t *Tracker) Run() error { } } - // Unregister packages not available anymore - if len(packagesAvailable) > 0 && !t.r.PackagesDeletionProtection { + // Unregister packages not available anymore or ignored. An empty list of + // packages available is not trusted to unregister packages (it may be the + // result of a transient issue), but ignored packages are always unregistered + // as this is an explicit request from the repository owner. + if !t.r.PackagesDeletionProtection { for key := range packagesRegistered { // Return ASAP if context is cancelled select { @@ -136,8 +141,9 @@ func (t *Tracker) Run() error { // Unregister pkg if it's not available anymore or if it's ignored name, version := pkg.ParseKey(key) - _, ok := packagesAvailable[key] - if !ok || shouldIgnorePackage(md, name, version) { + _, available := packagesAvailable[key] + ignored := md.IgnoresPackage(name, version) + if (len(packagesAvailable) > 0 && !available) || ignored { t.logger.Debug().Str("name", name).Str("v", version).Msg("unregistering package") p := &hub.Package{ Name: name, diff --git a/internal/tracker/tracker_test.go b/internal/tracker/tracker_test.go index 449e369eb1..f27b0c76b1 100644 --- a/internal/tracker/tracker_test.go +++ b/internal/tracker/tracker_test.go @@ -452,6 +452,93 @@ func TestTracker(t *testing.T) { sw.assertExpectations(t) }) + t.Run("only ignored packages unregistered when no packages are available", func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + sw := newServicesWrapper() + sw.rm.On("GetRemoteDigest", sw.svc.Ctx, r1).Return("", nil) + sw.ec.On("Init", r1.RepositoryID) + sw.rm.On("GetMetadata", r1, "").Return(&hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{ + { + Name: p1v1.Name, + Version: `^1\.0\.0$`, + }, + }, + }, nil) + sw.rm.On("GetPackagesDigest", sw.svc.Ctx, r1.RepositoryID).Return(map[string]string{ + pkg.BuildKey(p1v1): "", + pkg.BuildKey(p1v2): "", + }, nil) + sw.src.On("GetPackagesAvailable").Return(map[string]*hub.Package{}, nil) + sw.pm.On("Unregister", sw.svc.Ctx, p1v1).Return(nil) + + // Run test and check expectations + err := New(sw.svc, r1, zerolog.Nop()).Run() + assert.Nil(t, err) + sw.assertExpectations(t) + }) + + t.Run("ignored package not unregistered when deletion protection is enabled", func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + r := &hub.Repository{ + RepositoryID: r1.RepositoryID, + Kind: r1.Kind, + URL: r1.URL, + PackagesDeletionProtection: true, + } + sw := newServicesWrapper() + sw.rm.On("GetRemoteDigest", sw.svc.Ctx, r).Return("", nil) + sw.ec.On("Init", r.RepositoryID) + sw.rm.On("GetMetadata", r, "").Return(&hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{ + { + Name: p1v1.Name, + }, + }, + }, nil) + sw.rm.On("GetPackagesDigest", sw.svc.Ctx, r.RepositoryID).Return(map[string]string{ + pkg.BuildKey(p1v1): "", + }, nil) + sw.src.On("GetPackagesAvailable").Return(map[string]*hub.Package{}, nil) + + // Run test and check expectations + err := New(sw.svc, r, zerolog.Nop()).Run() + assert.Nil(t, err) + sw.assertExpectations(t) + }) + + t.Run("repository metadata provided to tracker source", func(t *testing.T) { + t.Parallel() + + // Setup services and expectations + md := &hub.RepositoryMetadata{ + Ignore: []*hub.RepositoryIgnoreEntry{ + { + Name: p1v1.Name, + }, + }, + } + sw := newServicesWrapper() + sw.svc.SetupTrackerSource = func(i *hub.TrackerSourceInput) hub.TrackerSource { + assert.Same(t, md, i.RepositoryMetadata) + return sw.src + } + sw.rm.On("GetRemoteDigest", sw.svc.Ctx, r1).Return("", nil) + sw.ec.On("Init", r1.RepositoryID) + sw.rm.On("GetMetadata", r1, "").Return(md, nil) + sw.rm.On("GetPackagesDigest", sw.svc.Ctx, r1.RepositoryID).Return(nil, nil) + sw.src.On("GetPackagesAvailable").Return(map[string]*hub.Package{}, nil) + + // Run test and check expectations + err := New(sw.svc, r1, zerolog.Nop()).Run() + assert.Nil(t, err) + sw.assertExpectations(t) + }) + t.Run("error setting verified publisher flag", func(t *testing.T) { t.Parallel()