Skip ignored packages in tracker sources

docs/metadata/artifacthub-repo.yml

@@ -8,14 +8,40 @@
 # changed if the `index.yaml` file changes. For git based repositories, it does
 # when the hash of the last commit in the branch you set up changes. This does
 # NOT apply to ownership claim operations, which are processed immediately.
-#
-repositoryID: The ID of the Artifact Hub repository where the packages will be published to (optional, but it enables verified publisher)
+
+# ID of the Artifact Hub repository where the packages will be published to.
+# It is displayed on the repository's card (ID) in the repositories tab of the
+# Artifact Hub control panel. The next time the repository is processed, the
+# repository will be marked as a verified publisher if the ID matches.
+repositoryID: 7a5ac6a4-1a1c-4d2e-9b8f-3f5e2c1d0b9a # (optional, but it enables verified publisher)
+
+# Repository owners, who can claim the ownership of the repository from the
+# Artifact Hub control panel. The user requesting the ownership claim must be
+# listed here, and the email must match the one used to sign in to Artifact
+# Hub. The repositoryID field can be omitted when claiming the ownership, as
+# it isn't known yet.
 owners: # (optional, used to claim repository ownership)
   - name: user1
     email: user1@email.com
   - name: user2
     email: user2@email.com
+
+# Ignored packages are not indexed and Artifact Hub skips as much of their
+# processing as possible, so most errors related to them won't be reported.
+# Some errors may still be reported for them, such as those found while
+# reading their metadata files or preparing their content. Ignored packages
+# already indexed will be removed.
+#
+# The version is a regular expression matched against the package version as
+# displayed in Artifact Hub (e.g. Helm chart versions are normalized, so `v1.0`
+# becomes `1.0.0`). It is not anchored, so `1.0.0` also matches `11.0.0` or
+# `1.0.0-rc.1`: use `^` and `$` to ignore specific versions only. A package
+# can be listed multiple times. For Helm charts and container images, errors
+# found while preparing ignored versions are not reported, so this can be used
+# to stop errors in old versions that can't be fixed from being reported.
 ignore: # (optional, packages that should not be indexed by Artifact Hub)
   - name: package1
   - name: package2 # Exact match
     version: beta # Regular expression (when omitted, all versions are ignored)
+  - name: package3
+    version: ^(1\.0\.0|1\.0\.1)$ # Only versions 1.0.0 and 1.0.1 are ignored

internal/hub/repo.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"regexp"
 
 	helmrepo "helm.sh/helm/v3/pkg/repo"
 )
@@ -355,6 +356,28 @@ type RepositoryMetadata struct {
 	Ignore       []*RepositoryIgnoreEntry `yaml:"ignore,omitempty"`
 }
 
+// IgnoresPackage checks if the package version provided must be ignored
+// according to the ignore list in the repository metadata. It is safe to
+// call it on a nil RepositoryMetadata.
+func (md *RepositoryMetadata) IgnoresPackage(name, version string) bool {
+	if md == nil {
+		return false
+	}
+	for _, e := range md.Ignore {
+		if e == nil || e.Name != name {
+			continue
+		}
+		if version == "" {
+			return true
+		}
+		matched, err := regexp.MatchString(e.Version, version)
+		if err == nil && matched {
+			return true
+		}
+	}
+	return false
+}
+
 // RepositoryIgnoreEntry represents an entry in the ignore list. This list is
 // meant to be used as a way to exclude packages from being indexed by Artifact
 // Hub. The name corresponds to a package name, and it must be an exact match.

internal/hub/repo_test.go

@@ -0,0 +1,129 @@
+package hub
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRepositoryMetadataIgnoresPackage(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		desc           string
+		md             *RepositoryMetadata
+		name           string
+		version        string
+		expectedResult bool
+	}{
+		{
+			"name only entry, empty version",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1"}}},
+			"pkg1",
+			"",
+			true,
+		},
+		{
+			"name only entry, any version",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1"}}},
+			"pkg1",
+			"1.0.0",
+			true,
+		},
+		{
+			"different name",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg2"}}},
+			"pkg1",
+			"1.0.0",
+			false,
+		},
+		{
+			"empty ignore list",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{}},
+			"pkg1",
+			"1.0.0",
+			false,
+		},
+		{
+			"nil metadata",
+			nil,
+			"pkg1",
+			"1.0.0",
+			false,
+		},
+		{
+			"unanchored regex matches prerelease",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "beta"}}},
+			"pkg1",
+			"1.0.0-beta1",
+			true,
+		},
+		{
+			"unanchored regex does not match",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "beta"}}},
+			"pkg1",
+			"1.0.0",
+			false,
+		},
+		{
+			"same version",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "1.0.0"}}},
+			"pkg1",
+			"1.0.0",
+			true,
+		},
+		{
+			"different version",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "1.0.0"}}},
+			"pkg1",
+			"1.0.1",
+			false,
+		},
+		{
+			"only second entry matches",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{
+				{Name: "pkg1", Version: "^2\\.0\\.0$"},
+				{Name: "pkg1", Version: "^1\\.0\\.0$"},
+			}},
+			"pkg1",
+			"1.0.0",
+			true,
+		},
+		{
+			"anchored regex matches",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "^(1\\.0\\.0|1\\.0\\.1)$"}}},
+			"pkg1",
+			"1.0.1",
+			true,
+		},
+		{
+			"anchored regex does not match a longer version",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "^1\\.0\\.0$"}}},
+			"pkg1",
+			"11.0.0",
+			false,
+		},
+		{
+			"invalid regex",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{{Name: "pkg1", Version: "["}}},
+			"pkg1",
+			"1.0.0",
+			false,
+		},
+		{
+			"nil entry",
+			&RepositoryMetadata{Ignore: []*RepositoryIgnoreEntry{nil}},
+			"pkg1",
+			"1.0.0",
+			false,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			t.Parallel()
+
+			result := tc.md.IgnoresPackage(tc.name, tc.version)
+			assert.Equal(t, tc.expectedResult, result)
+		})
+	}
+}

internal/hub/tracker.go

@@ -45,10 +45,13 @@ type TrackerSource interface {
 }
 
 // TrackerSourceInput represents the input provided to a TrackerSource to get
-// the packages available in a repository when tracking it.
+// the packages available in a repository when tracking it. Sources should skip
+// the package versions ignored by the repository metadata provided, and must
+// treat that metadata as read-only.
 type TrackerSourceInput struct {
 	Repository         *Repository
 	RepositoryDigest   string
+	RepositoryMetadata *RepositoryMetadata
 	PackagesRegistered map[string]string
 	BasePath           string
 	Svc                *TrackerSourceServices

internal/repo/errors.go

@@ -11,8 +11,10 @@ import (
 )
 
 const (
-	// maxErrorsPerRepository represents the maximum number of errors we want
-	// to collect for a given repository.
+	// maxErrorsPerRepository represents the maximum number of errors stored
+	// for a given repository. It is applied after sorting the errors, so the
+	// stored text only depends on the errors produced, not on the order they
+	// were collected in.
 	maxErrorsPerRepository = 100
 )
 
@@ -52,9 +54,7 @@ func (c *ErrorsCollector) Append(repositoryID string, err string) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	if len(c.errors[repositoryID]) < maxErrorsPerRepository {
-		c.errors[repositoryID] = append(c.errors[repositoryID], err)
-	}
+	c.errors[repositoryID] = append(c.errors[repositoryID], err)
 }
 
 // Flush aggregates all errors collected per repository as a single text and
@@ -68,8 +68,12 @@ func (c *ErrorsCollector) Flush() {
 		// a repository concurrently, and the order the errors are produced is
 		// not guaranteed. In order to be able to notify users when something
 		// goes wrong during repositories tracking or scanning, we need to be
-		// able to compare the errors produced among executions.
+		// able to compare the errors produced among executions. For the same
+		// reason, errors are truncated only after sorting them.
 		sort.Strings(errors)
+		if len(errors) > maxErrorsPerRepository {
+			errors = errors[:maxErrorsPerRepository]
+		}
 
 		var allErrors strings.Builder
 		for i, err := range errors {

internal/repo/errors_test.go

@@ -2,6 +2,8 @@ package repo
 
 import (
 	"context"
+	"fmt"
+	"strings"
 	"testing"
 )
 
@@ -44,3 +46,53 @@ func TestCollector(t *testing.T) {
 		})
 	}
 }
+
+func TestCollectorTruncation(t *testing.T) {
+	// Expected text: the first maxErrorsPerRepository errors once sorted
+	expectedErrors := make([]string, 0, maxErrorsPerRepository)
+	for i := range maxErrorsPerRepository {
+		expectedErrors = append(expectedErrors, fmt.Sprintf("error%03d", i))
+	}
+	expectedText := strings.Join(expectedErrors, "\n")
+
+	kinds := []struct {
+		kind         ErrorsCollectorKind
+		expectedCall string
+	}{
+		{Scanner, "SetLastScanningResults"},
+		{Tracker, "SetLastTrackingResults"},
+	}
+	orders := []struct {
+		name       string
+		descending bool
+	}{
+		{"ascending", false},
+		{"descending", true},
+	}
+	for _, k := range kinds {
+		for _, o := range orders {
+			t.Run(fmt.Sprintf("%s %s", k.expectedCall, o.name), func(t *testing.T) {
+				t.Parallel()
+
+				// Setup errors collector
+				rm := &ManagerMock{}
+				ec := NewErrorsCollector(rm, k.kind)
+
+				// Append more errors than the maximum allowed
+				const numErrors = 150
+				for i := range numErrors {
+					n := i
+					if o.descending {
+						n = numErrors - 1 - i
+					}
+					ec.Append("repo1", fmt.Sprintf("error%03d", n))
+				}
+
+				// Flush errors and check only the first ones were stored
+				rm.On(k.expectedCall, context.Background(), "repo1", expectedText).Return(nil)
+				ec.Flush()
+				rm.AssertExpectations(t)
+			})
+		}
+	}
+}

internal/tracker/helpers.go

@@ -3,7 +3,6 @@ package tracker
 import (
 	"context"
 	"fmt"
-	"regexp"
 
 	"github.com/artifacthub/hub/internal/hub"
 	"github.com/artifacthub/hub/internal/tracker/source/container"
@@ -161,35 +160,3 @@ func setVerifiedPublisherFlag(
 	}
 	return nil
 }
-
-// shouldIgnorePackage checks if the package provided should be ignored.
-func shouldIgnorePackage(md *hub.RepositoryMetadata, name, version string) bool {
-	if md == nil {
-		return false
-	}
-	for _, ignoreEntry := range md.Ignore {
-		if matchesEntry(ignoreEntry, name, version) {
-			return true
-		}
-	}
-	return false
-}
-
-// matchesEntry checks if the package name and version provide match a given
-// ignore entry.
-func matchesEntry(ignoreEntry *hub.RepositoryIgnoreEntry, name, version string) bool {
-	if ignoreEntry.Name != name {
-		return false
-	}
-	if version == "" {
-		return true
-	}
-	versionMatch, err := regexp.Match(ignoreEntry.Version, []byte(version))
-	if err != nil {
-		return false
-	}
-	if versionMatch {
-		return true
-	}
-	return false
-}