diff --git a/.github/workflows/tang-actions.yaml b/.github/workflows/tang-actions.yaml index e9bbc5a..bde4145 100644 --- a/.github/workflows/tang-actions.yaml +++ b/.github/workflows/tang-actions.yaml @@ -68,5 +68,7 @@ jobs: DATABASE_NAME: pulp DATABASE_PASSWORD: password SERVER_URL: http://localhost:8087 + SERVER_CONTENT_URL: http://localhost:8088 + SERVER_CONTENT_PATH_PREFIX: /pulp/content/ SERVER_USERNAME: admin SERVER_PASSWORD: password diff --git a/README.md b/README.md index 5c4a766..22f11e2 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ CONFIG_PATH="$(pwd)/configs/" go test ./internal/test/integration/ -run TestMave CONFIG_PATH="$(pwd)/configs/" go test ./internal/test/integration/ -run TestNpmSuite -v ``` -The Python integration test syncs `shelf-reader` from PyPI into a random domain via the Pulp API, then asserts tangy can read it from the database. The Maven integration test pull-through caches `junit:junit:4.13.2` from a local nginx fixture in the compose stack (avoids Maven Central rate limits in CI), adds the cached content to a repository, then asserts tangy can read it from the database. The npm integration test syncs `is-odd@3.0.1` from registry.npmjs.org (version-specific metadata URL), then asserts tangy can read it from the database. Test data is left in the database after a run; use `make compose-clean` to wipe volumes and start fresh. +The Python integration test syncs `shelf-reader` from PyPI into a random domain via the Pulp API, then asserts tangy can read it from the database. The Maven integration test pull-through caches artifacts, adds the cached content to a repository, then asserts tangy can read it from the database. The npm integration test syncs `is-odd@3.0.1` from registry.npmjs.org (version-specific metadata URL), then asserts tangy can read it from the database. Test data is left in the database after a run; use `make compose-clean` to wipe volumes and start fresh. ### Mocking Tangy also exports a mock interface you can regenerate using the [mockery](https://github.com/vektra/mockery) tool. diff --git a/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00001/foo.bar.1.2.3.rhlw-00001.jar b/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00001/foo.bar.1.2.3.rhlw-00001.jar deleted file mode 100644 index 518a38a..0000000 Binary files a/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00001/foo.bar.1.2.3.rhlw-00001.jar and /dev/null differ diff --git a/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00001/foo.bar.1.2.3.rhlw-00001.pom b/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00001/foo.bar.1.2.3.rhlw-00001.pom deleted file mode 100644 index 6130be7..0000000 --- a/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00001/foo.bar.1.2.3.rhlw-00001.pom +++ /dev/null @@ -1,7 +0,0 @@ - - - 4.0.0 - foo - bar - 1.2.3.rhlw-00001 - diff --git a/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00002/foo.bar.1.2.3.rhlw-00002.jar b/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00002/foo.bar.1.2.3.rhlw-00002.jar deleted file mode 100644 index 518a38a..0000000 Binary files a/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00002/foo.bar.1.2.3.rhlw-00002.jar and /dev/null differ diff --git a/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00002/foo.bar.1.2.3.rhlw-00002.pom b/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00002/foo.bar.1.2.3.rhlw-00002.pom deleted file mode 100644 index b97e0aa..0000000 --- a/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3.rhlw-00002/foo.bar.1.2.3.rhlw-00002.pom +++ /dev/null @@ -1,7 +0,0 @@ - - - 4.0.0 - foo - bar - 1.2.3.rhlw-00002 - diff --git a/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1.rhlw-00001/foo.bar.2.0.1.rhlw-00001.jar b/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1.rhlw-00001/foo.bar.2.0.1.rhlw-00001.jar deleted file mode 100644 index 97203b3..0000000 Binary files a/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1.rhlw-00001/foo.bar.2.0.1.rhlw-00001.jar and /dev/null differ diff --git a/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1.rhlw-00001/foo.bar.2.0.1.rhlw-00001.pom b/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1.rhlw-00001/foo.bar.2.0.1.rhlw-00001.pom deleted file mode 100644 index c82bb4b..0000000 --- a/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1.rhlw-00001/foo.bar.2.0.1.rhlw-00001.pom +++ /dev/null @@ -1,7 +0,0 @@ - - - 4.0.0 - foo - bar - 2.0.1.rhlw-00001 - diff --git a/compose_files/pulp/docker-compose.yml b/compose_files/pulp/docker-compose.yml index d755f0c..64d5536 100644 --- a/compose_files/pulp/docker-compose.yml +++ b/compose_files/pulp/docker-compose.yml @@ -54,12 +54,6 @@ services: healthcheck: test: ["CMD", "redis-cli", "--raw", "incr", "ping"] - maven_fixture: - image: "docker.io/library/nginx:alpine" - volumes: - - "./assets/maven-fixture:/usr/share/nginx/html:ro" - restart: always - pulp_api: image: "quay.io/redhat-services-prod/pulp-services-tenant/pulp:latest" platform: linux/amd64 diff --git a/configs/config.yaml.example b/configs/config.yaml.example index 11f8c4a..e503cfa 100644 --- a/configs/config.yaml.example +++ b/configs/config.yaml.example @@ -9,7 +9,9 @@ database: # Configuration options for the pulp server server: url: "http://localhost:8087" + content_url: "http://localhost:8088" username: "admin" password: "password" storage_type: "local" - download_policy: "on_demand" \ No newline at end of file + download_policy: "on_demand" + content_path_prefix: "/api/pulp-content/" \ No newline at end of file diff --git a/internal/config/config.go b/internal/config/config.go index 1bc5a53..00cf618 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -48,19 +48,23 @@ func setDefaults(v *viper.Viper) { v.SetDefault("database.name", "") v.SetDefault("server.url", "") + v.SetDefault("server.content_url", "") v.SetDefault("server.username", "") v.SetDefault("server.password", "") v.SetDefault("server.storage_type", "") v.SetDefault("server.download_policy", "") + v.SetDefault("server.content_path_prefix", "/api/pulp-content/") } // Server configuration options for connecting to a pulp server type Server struct { - Url string `mapstructure:"url"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - StorageType string `mapstructure:"storage_type"` - DownloadPolicy string `mapstructure:"download_policy"` + Url string `mapstructure:"url"` + ContentUrl string `mapstructure:"content_url"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + StorageType string `mapstructure:"storage_type"` + DownloadPolicy string `mapstructure:"download_policy"` + ContentPathPrefix string `mapstructure:"content_path_prefix"` } // Database configuration options for connection to a pulp database. Duplicated of tangy.Database. diff --git a/internal/test/integration/maven_test.go b/internal/test/integration/maven_test.go index 8534b79..2da3d88 100644 --- a/internal/test/integration/maven_test.go +++ b/internal/test/integration/maven_test.go @@ -16,32 +16,34 @@ import ( ) const ( - testMavenRepoName = "foo-fixture" - testMavenDistributionBasePath = "foo-fixture" - testMavenRemoteURL = "http://maven_fixture/" - testMavenContentOrigin = "http://localhost:8088" - testMavenGroupID = "foo" - testMavenArtifactID = "bar" - testMavenBuildVersion123R1 = "1.2.3.rhlw-00001" - testMavenBuildVersion123R2 = "1.2.3.rhlw-00002" - testMavenBuildVersion201R1 = "2.0.1.rhlw-00001" + testMavenRepoName = "maven-releases-fixture" + testMavenDistributionBasePath = "maven-releases-fixture" + testMavenFixtureUrl = "https://content-services.github.io/fixtures/maven/maven-releases/" + testMavenGroupID = "com.example.fixture" + testMavenArtifactID = "raccoon" + testMavenBaseVersion100 = "1.0.0" + testMavenBaseVersion200 = "2.0.0" ) var testMavenArtifactPaths = []string{ - "foo/bar/1.2.3.rhlw-00001/foo.bar.1.2.3.rhlw-00001.pom", - "foo/bar/1.2.3.rhlw-00001/foo.bar.1.2.3.rhlw-00001.jar", - "foo/bar/1.2.3.rhlw-00002/foo.bar.1.2.3.rhlw-00002.pom", - "foo/bar/1.2.3.rhlw-00002/foo.bar.1.2.3.rhlw-00002.jar", - "foo/bar/2.0.1.rhlw-00001/foo.bar.2.0.1.rhlw-00001.pom", - "foo/bar/2.0.1.rhlw-00001/foo.bar.2.0.1.rhlw-00001.jar", + "com/example/fixture/raccoon/1.0.0.rhlw-00001/com.example.fixture.raccoon.1.0.0.rhlw-00001.pom", + "com/example/fixture/raccoon/1.0.0.rhlw-00001/com.example.fixture.raccoon.1.0.0.rhlw-00001.jar", + "com/example/fixture/raccoon/1.0.0.rhlw-00002/com.example.fixture.raccoon.1.0.0.rhlw-00002.pom", + "com/example/fixture/raccoon/1.0.0.rhlw-00002/com.example.fixture.raccoon.1.0.0.rhlw-00002.jar", + "com/example/fixture/raccoon/2.0.0.rhlw-00001/com.example.fixture.raccoon.2.0.0.rhlw-00001.pom", + "com/example/fixture/raccoon/2.0.0.rhlw-00001/com.example.fixture.raccoon.2.0.0.rhlw-00001.jar", + "com/example/fixture/raccoon/2.0.0.rhlw-00002/com.example.fixture.raccoon.2.0.0.rhlw-00002.pom", + "com/example/fixture/raccoon/2.0.0.rhlw-00002/com.example.fixture.raccoon.2.0.0.rhlw-00002.jar", } type MavenSuite struct { suite.Suite - client *zestwrapper.MavenZest - tangy tangy.Tangy - domainName string - repositoryHref string + client *zestwrapper.MavenZest + tangy tangy.Tangy + domainName string + contentUrl string + contentPathPrefix string + repositoryHref string } func (m *MavenSuite) createTestRepository(t *testing.T) { @@ -51,13 +53,13 @@ func (m *MavenSuite) createTestRepository(t *testing.T) { repoHref, remoteHref, err := m.client.CreateRepository( m.domainName, testMavenRepoName, - testMavenRemoteURL, + testMavenFixtureUrl, testMavenDistributionBasePath, ) require.NoError(t, err) for _, artifactPath := range testMavenArtifactPaths { - err = m.client.FetchArtifact(testMavenContentOrigin, m.domainName, testMavenDistributionBasePath, artifactPath) + err = m.client.FetchArtifact(m.contentUrl, m.contentPathPrefix, m.domainName, testMavenDistributionBasePath, artifactPath) require.NoError(t, err) } @@ -88,6 +90,8 @@ func TestMavenSuite(t *testing.T) { m := MavenSuite{} m.client = &mavenZest m.tangy = ta + m.contentUrl = s.ContentUrl + m.contentPathPrefix = s.ContentPathPrefix m.domainName = RandStringBytes(10) m.createTestRepository(t) @@ -107,23 +111,22 @@ func (m *MavenSuite) TestMavenPackageList() { pkg := response.Results[0] assert.Equal(m.T(), testMavenGroupID, pkg.GroupID) assert.Equal(m.T(), testMavenArtifactID, pkg.ArtifactID) - assert.Contains(m.T(), pkg.Versions, testMavenBuildVersion123R1) - assert.Contains(m.T(), pkg.Versions, testMavenBuildVersion123R2) - assert.Contains(m.T(), pkg.Versions, testMavenBuildVersion201R1) - require.NotEmpty(m.T(), pkg.LatestReleases) + assert.Len(m.T(), pkg.Versions, 2) + assert.Contains(m.T(), pkg.Versions, testMavenBaseVersion100) + assert.Contains(m.T(), pkg.Versions, testMavenBaseVersion200) + require.Len(m.T(), pkg.LatestReleases, 2) - foundVersions := make(map[string]bool) + foundVersions := make(map[string]string) for _, latest := range pkg.LatestReleases { - foundVersions[latest.Version] = true + foundVersions[latest.Version] = latest.Release assert.NotEmpty(m.T(), latest.CreatedAt) } - assert.True(m.T(), foundVersions[testMavenBuildVersion123R1]) - assert.True(m.T(), foundVersions[testMavenBuildVersion123R2]) - assert.True(m.T(), foundVersions[testMavenBuildVersion201R1]) + assert.Equal(m.T(), "rhlw-00002", foundVersions[testMavenBaseVersion100]) + assert.Equal(m.T(), "rhlw-00002", foundVersions[testMavenBaseVersion200]) } func (m *MavenSuite) TestMavenPackageListSearchFilter() { - response, err := m.tangy.MavenPackageList(context.Background(), m.repositoryHref, tangy.MavenPackageListFilters{Search: "foo"}, tangy.PageOptions{ + response, err := m.tangy.MavenPackageList(context.Background(), m.repositoryHref, tangy.MavenPackageListFilters{Search: "raccoon"}, tangy.PageOptions{ Offset: 0, Limit: 10, }) @@ -167,63 +170,72 @@ func (m *MavenSuite) TestMavenPackageListEmptyHref() { assert.Zero(m.T(), response.Total) } -func (m *MavenSuite) TestMavenBuildList() { - response, err := m.tangy.MavenBuildList( +func (m *MavenSuite) TestMavenVersionsList() { + response, err := m.tangy.MavenVersionsList( context.Background(), m.repositoryHref, testMavenGroupID, testMavenArtifactID, - testMavenBuildVersion123R1, + testMavenBaseVersion100, tangy.PageOptions{Offset: 0, Limit: 10}, ) require.NoError(m.T(), err) - require.NotEmpty(m.T(), response.Results) + require.Len(m.T(), response.Results, 1) assert.Equal(m.T(), 1, response.Total) assert.Equal(m.T(), 10, response.Limit) - build := response.Results[0] - assert.Equal(m.T(), testMavenGroupID, build.GroupID) - assert.Equal(m.T(), testMavenArtifactID, build.ArtifactID) - assert.Equal(m.T(), testMavenBuildVersion123R1, build.Version) - assert.True(m.T(), strings.HasSuffix(build.Filename, ".pom"), build.Filename) - assert.NotEmpty(m.T(), build.CreatedAt) + version := response.Results[0] + assert.Equal(m.T(), testMavenGroupID, version.GroupID) + assert.Equal(m.T(), testMavenArtifactID, version.ArtifactID) + assert.Equal(m.T(), testMavenBaseVersion100, version.Version) + require.Len(m.T(), version.Builds, 2) + + foundReleases := make(map[string]bool) + for _, build := range version.Builds { + assert.Equal(m.T(), testMavenBaseVersion100, build.Version) + assert.True(m.T(), strings.HasSuffix(build.Filename, ".pom"), build.Filename) + assert.NotEmpty(m.T(), build.CreatedAt) + foundReleases[build.Release] = true + } + assert.True(m.T(), foundReleases["rhlw-00001"]) + assert.True(m.T(), foundReleases["rhlw-00002"]) } -func (m *MavenSuite) TestMavenBuildListPagination() { - response, err := m.tangy.MavenBuildList( +func (m *MavenSuite) TestMavenVersionsListPagination() { + response, err := m.tangy.MavenVersionsList( context.Background(), m.repositoryHref, testMavenGroupID, testMavenArtifactID, - testMavenBuildVersion123R1, + "", tangy.PageOptions{Offset: 0, Limit: 1}, ) require.NoError(m.T(), err) assert.Len(m.T(), response.Results, 1) - assert.Equal(m.T(), 1, response.Total) + assert.Equal(m.T(), 2, response.Total) - response, err = m.tangy.MavenBuildList( + response, err = m.tangy.MavenVersionsList( context.Background(), m.repositoryHref, testMavenGroupID, testMavenArtifactID, - testMavenBuildVersion123R1, - tangy.PageOptions{Offset: 1, Limit: 10}, + "", + tangy.PageOptions{Offset: 2, Limit: 10}, ) require.NoError(m.T(), err) assert.Empty(m.T(), response.Results) - assert.Equal(m.T(), 1, response.Total) + assert.Equal(m.T(), 2, response.Total) } -func (m *MavenSuite) TestMavenBuildListEmptyHref() { - response, err := m.tangy.MavenBuildList(context.Background(), "", testMavenGroupID, testMavenArtifactID, testMavenBuildVersion123R1, tangy.PageOptions{Limit: 10}) +func (m *MavenSuite) TestMavenVersionsListEmptyHref() { + response, err := m.tangy.MavenVersionsList(context.Background(), "", testMavenGroupID, testMavenArtifactID, testMavenBaseVersion100, tangy.PageOptions{Limit: 10}) require.NoError(m.T(), err) assert.Empty(m.T(), response.Results) assert.Zero(m.T(), response.Total) } -func (m *MavenSuite) TestMavenBuildListOptionalFilters() { - response, err := m.tangy.MavenBuildList( +func (m *MavenSuite) TestMavenVersionsListOptionalFilters() { + response, err := m.tangy.MavenVersionsList( context.Background(), m.repositoryHref, "", @@ -232,14 +244,24 @@ func (m *MavenSuite) TestMavenBuildListOptionalFilters() { tangy.PageOptions{Offset: 0, Limit: 10}, ) require.NoError(m.T(), err) - require.NotEmpty(m.T(), response.Results) - assert.Equal(m.T(), 3, response.Total) - - build := response.Results[0] - assert.Equal(m.T(), testMavenGroupID, build.GroupID) - assert.Equal(m.T(), testMavenArtifactID, build.ArtifactID) + require.Len(m.T(), response.Results, 2) + assert.Equal(m.T(), 2, response.Total) + + type versionRelease struct{ version, release string } + foundBuilds := make(map[versionRelease]bool) + for _, result := range response.Results { + assert.Equal(m.T(), testMavenGroupID, result.GroupID) + assert.Equal(m.T(), testMavenArtifactID, result.ArtifactID) + for _, build := range result.Builds { + foundBuilds[versionRelease{build.Version, build.Release}] = true + } + } + assert.True(m.T(), foundBuilds[versionRelease{testMavenBaseVersion100, "rhlw-00001"}]) + assert.True(m.T(), foundBuilds[versionRelease{testMavenBaseVersion100, "rhlw-00002"}]) + assert.True(m.T(), foundBuilds[versionRelease{testMavenBaseVersion200, "rhlw-00001"}]) + assert.True(m.T(), foundBuilds[versionRelease{testMavenBaseVersion200, "rhlw-00002"}]) - filtered, err := m.tangy.MavenBuildList( + filtered, err := m.tangy.MavenVersionsList( context.Background(), m.repositoryHref, testMavenGroupID, @@ -248,10 +270,10 @@ func (m *MavenSuite) TestMavenBuildListOptionalFilters() { tangy.PageOptions{Offset: 0, Limit: 10}, ) require.NoError(m.T(), err) - require.Len(m.T(), filtered.Results, 3) - assert.Equal(m.T(), 3, filtered.Total) + require.Len(m.T(), filtered.Results, 2) + assert.Equal(m.T(), 2, filtered.Total) - filtered, err = m.tangy.MavenBuildList( + filtered, err = m.tangy.MavenVersionsList( context.Background(), m.repositoryHref, "", @@ -260,15 +282,15 @@ func (m *MavenSuite) TestMavenBuildListOptionalFilters() { tangy.PageOptions{Offset: 0, Limit: 10}, ) require.NoError(m.T(), err) - require.Len(m.T(), filtered.Results, 3) - assert.Equal(m.T(), 3, filtered.Total) + require.Len(m.T(), filtered.Results, 2) + assert.Equal(m.T(), 2, filtered.Total) } func (m *MavenSuite) TestMavenRepositoryMetrics() { metrics, err := m.tangy.MavenRepositoryMetrics(context.Background(), m.repositoryHref) require.NoError(m.T(), err) assert.Equal(m.T(), 1, metrics.PackageCount) - assert.Equal(m.T(), 3, metrics.BuildCount) + assert.Equal(m.T(), 4, metrics.BuildCount) assert.Equal(m.T(), 2, metrics.VersionCount) } diff --git a/internal/zestwrapper/maven.go b/internal/zestwrapper/maven.go index 6742981..7925998 100644 --- a/internal/zestwrapper/maven.go +++ b/internal/zestwrapper/maven.go @@ -123,9 +123,10 @@ func (m *MavenZest) CreateRepository(domain, name, remoteURL, distributionBasePa } // FetchArtifact triggers pull-through caching for an artifact path via the content app. -func (m *MavenZest) FetchArtifact(contentOrigin, domain, distributionBasePath, artifactPath string) error { +func (m *MavenZest) FetchArtifact(contentOrigin, contentPathPrefix, domain, distributionBasePath, artifactPath string) error { artifactPath = strings.TrimPrefix(artifactPath, "/") - url := fmt.Sprintf("%s/pulp/content/%s/%s/%s", strings.TrimRight(contentOrigin, "/"), domain, distributionBasePath, artifactPath) + contentPathPrefix = strings.Trim(contentPathPrefix, "/") + url := fmt.Sprintf("%s/%s/%s/%s/%s", strings.TrimRight(contentOrigin, "/"), contentPathPrefix, domain, distributionBasePath, artifactPath) resp, err := m.httpClient.Get(url) if err != nil { diff --git a/pkg/tangy/interface.go b/pkg/tangy/interface.go index 33a4ffd..a6de038 100644 --- a/pkg/tangy/interface.go +++ b/pkg/tangy/interface.go @@ -72,7 +72,7 @@ type Tangy interface { PythonBuildList(ctx context.Context, repositoryHref, nameNormalized, version string, pageOpts PageOptions) (PythonBuildListResponse, error) PythonRepositoryMetrics(ctx context.Context, repositoryHref string) (PythonRepositoryMetrics, error) MavenPackageList(ctx context.Context, repositoryHref string, filterOpts MavenPackageListFilters, pageOpts PageOptions) (MavenPackageListResponse, error) - MavenBuildList(ctx context.Context, repositoryHref, groupID, artifactID, version string, pageOpts PageOptions) (MavenBuildListResponse, error) + MavenVersionsList(ctx context.Context, repositoryHref, groupID, artifactID, version string, pageOpts PageOptions) (MavenVersionsResponse, error) MavenRepositoryMetrics(ctx context.Context, repositoryHref string) (MavenRepositoryMetrics, error) NpmPackageList(ctx context.Context, repositoryHref string, filterOpts NpmPackageListFilters, pageOpts PageOptions) (NpmPackageListResponse, error) NpmPackageGet(ctx context.Context, repositoryHref, name, version string) (NpmPackageDetail, error) diff --git a/pkg/tangy/maven.go b/pkg/tangy/maven.go index 99ab33b..b3e748a 100644 --- a/pkg/tangy/maven.go +++ b/pkg/tangy/maven.go @@ -36,20 +36,25 @@ type MavenPackageListResponse struct { Offset int `json:"offset"` } -type MavenBuildListItem struct { - GroupID string `json:"group_id"` - ArtifactID string `json:"artifact_id"` - Version string `json:"version"` - Release string `json:"release"` - Filename string `json:"filename"` - CreatedAt string `json:"created_at"` +type MavenBuildInfo struct { + Version string `json:"version"` + Release string `json:"release"` + Filename string `json:"filename"` + CreatedAt string `json:"created_at"` } -type MavenBuildListResponse struct { - Results []MavenBuildListItem `json:"results"` - Total int `json:"total"` - Limit int `json:"limit"` - Offset int `json:"offset"` +type MavenVersionsItem struct { + GroupID string `json:"group_id"` + ArtifactID string `json:"artifact_id"` + Version string `json:"version"` + Builds []MavenBuildInfo `json:"builds"` +} + +type MavenVersionsResponse struct { + Results []MavenVersionsItem `json:"results"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` } type MavenRepositoryMetrics struct { @@ -58,14 +63,6 @@ type MavenRepositoryMetrics struct { VersionCount int `json:"version_count"` } -type mavenArtifactQueryResult struct { - GroupID string - ArtifactID string - Version string - Filename string - CreatedAt time.Time -} - // MavenPackageList lists Maven packages from the latest version of a repository, grouped by group_id and artifact_id // Only includes artifacts with .pom files func (t *tangyImpl) MavenPackageList(ctx context.Context, repositoryHref string, filterOpts MavenPackageListFilters, pageOpts PageOptions) (MavenPackageListResponse, error) { @@ -138,10 +135,10 @@ func (t *tangyImpl) MavenPackageList(ctx context.Context, repositoryHref string, SELECT rp.group_id, rp.artifact_id, - rp.version, + regexp_replace(rp.version, '\.` + mavenReleaseQualifierPattern + `$', '') as base_version, rp.filename, cc.pulp_created, - ROW_NUMBER() OVER (PARTITION BY rp.group_id, rp.artifact_id, rp.version ORDER BY cc.pulp_created DESC) as rn + ROW_NUMBER() OVER (PARTITION BY rp.group_id, rp.artifact_id, regexp_replace(rp.version, '\.` + mavenReleaseQualifierPattern + `$', '') ORDER BY cc.pulp_created DESC) as rn FROM maven_mavenartifact rp INNER JOIN core_content cc ON rp.content_ptr_id = cc.pulp_id ` + innerUnion + artifactFilters + ` @@ -150,7 +147,7 @@ func (t *tangyImpl) MavenPackageList(ctx context.Context, repositoryHref string, SELECT group_id, artifact_id, - version, + base_version, filename, pulp_created FROM package_versions @@ -160,7 +157,7 @@ func (t *tangyImpl) MavenPackageList(ctx context.Context, repositoryHref string, SELECT group_id, artifact_id, - ARRAY_AGG(DISTINCT version ORDER BY version) as versions + ARRAY_AGG(DISTINCT base_version ORDER BY base_version) as versions FROM latest_per_version GROUP BY group_id, artifact_id ORDER BY group_id, artifact_id @@ -173,12 +170,12 @@ func (t *tangyImpl) MavenPackageList(ctx context.Context, repositoryHref string, COALESCE( JSON_AGG( JSON_BUILD_OBJECT( - 'version', lpv.version, + 'version', lpv.base_version, 'release', '', 'filename', lpv.filename, 'created_at', lpv.pulp_created - ) ORDER BY lpv.version - ) FILTER (WHERE lpv.version IS NOT NULL), + ) ORDER BY lpv.base_version + ) FILTER (WHERE lpv.base_version IS NOT NULL), '[]'::json ) as latest_releases_json FROM packages p @@ -310,16 +307,16 @@ func getLatestRepositoryVersion(ctx context.Context, conn *pgxpool.Conn, repoUUI return latestVersion, nil } -// MavenBuildList lists all Maven artifacts (builds), optionally filtered by group_id, artifact_id, and version +// MavenVersionsList lists all Maven artifacts (builds), optionally filtered by group_id, artifact_id, and version // from the latest version of a repository -func (t *tangyImpl) MavenBuildList(ctx context.Context, repositoryHref, groupID, artifactID, version string, pageOpts PageOptions) (MavenBuildListResponse, error) { +func (t *tangyImpl) MavenVersionsList(ctx context.Context, repositoryHref, groupID, artifactID, version string, pageOpts PageOptions) (MavenVersionsResponse, error) { if repositoryHref == "" { - return MavenBuildListResponse{}, nil + return MavenVersionsResponse{}, nil } conn, err := t.pool.Acquire(ctx) if err != nil { - return MavenBuildListResponse{}, err + return MavenVersionsResponse{}, err } defer conn.Release() @@ -330,13 +327,13 @@ func (t *tangyImpl) MavenBuildList(ctx context.Context, repositoryHref, groupID, // Parse repository UUID from href repoUUID, err := parseRepositoryHref(repositoryHref) if err != nil { - return MavenBuildListResponse{}, fmt.Errorf("error parsing repository href: %w", err) + return MavenVersionsResponse{}, fmt.Errorf("error parsing repository href: %w", err) } // Get the latest version for this repository latestVersion, err := getLatestRepositoryVersion(ctx, conn, repoUUID) if err != nil { - return MavenBuildListResponse{}, fmt.Errorf("error getting latest repository version: %w", err) + return MavenVersionsResponse{}, fmt.Errorf("error getting latest repository version: %w", err) } repoVerMap := []ParsedRepoVersion{{ @@ -358,64 +355,122 @@ func (t *tangyImpl) MavenBuildList(ctx context.Context, repositoryHref, groupID, } if version != "" { args["version"] = version - whereClause += "\n\t\tAND rp.version = @version" + whereClause += "\n\t\tAND regexp_replace(rp.version, '\\." + mavenReleaseQualifierPattern + "$', '') = @version" } innerUnion, err := contentIdsInVersions(ctx, conn, repoVerMap, &args) if err != nil { - return MavenBuildListResponse{}, err + return MavenVersionsResponse{}, err } - // Count query for total artifacts + pomFilter := ` + AND rp.filename LIKE '%.pom'` + + // Count query for total distinct versions countQuery := ` - SELECT COUNT(*) + SELECT COUNT(DISTINCT (rp.group_id, rp.artifact_id, regexp_replace(rp.version, '\.` + mavenReleaseQualifierPattern + `$', ''))) FROM maven_mavenartifact rp - ` + innerUnion + whereClause + ` - AND rp.filename LIKE '%.pom'` + ` + innerUnion + whereClause + pomFilter var countTotal int err = conn.QueryRow(ctx, countQuery, args).Scan(&countTotal) if err != nil { - return MavenBuildListResponse{}, err + return MavenVersionsResponse{}, err } - // Main query to get all matching artifacts - query := ` - SELECT rp.group_id, rp.artifact_id, rp.version, rp.filename, cc.pulp_created as created_at - FROM maven_mavenartifact rp - INNER JOIN core_content cc ON rp.content_ptr_id = cc.pulp_id - ` + innerUnion + whereClause + ` - AND rp.filename LIKE '%.pom' - ORDER BY cc.pulp_created DESC - LIMIT @limit OFFSET @offset` - args["limit"] = pageOpts.Limit args["offset"] = pageOpts.Offset + query := ` + WITH version_builds AS ( + SELECT + rp.group_id, + rp.artifact_id, + regexp_replace(rp.version, '\.` + mavenReleaseQualifierPattern + `$', '') as base_version, + rp.filename, + cc.pulp_created as created_at + FROM maven_mavenartifact rp + INNER JOIN core_content cc ON rp.content_ptr_id = cc.pulp_id + ` + innerUnion + whereClause + pomFilter + ` + ), + distinct_versions AS ( + SELECT + group_id, + artifact_id, + base_version, + MAX(created_at) as latest_created_at + FROM version_builds + GROUP BY group_id, artifact_id, base_version + ORDER BY latest_created_at DESC + LIMIT @limit OFFSET @offset + ) + SELECT + dv.group_id, + dv.artifact_id, + dv.base_version as version, + COALESCE( + JSON_AGG( + JSON_BUILD_OBJECT( + 'version', vb.base_version, + 'filename', vb.filename, + 'created_at', vb.created_at + ) ORDER BY vb.created_at DESC + ), + '[]'::json + ) as builds_json + FROM distinct_versions dv + INNER JOIN version_builds vb ON dv.group_id = vb.group_id AND dv.artifact_id = vb.artifact_id AND dv.base_version = vb.base_version + GROUP BY dv.group_id, dv.artifact_id, dv.base_version, dv.latest_created_at + ORDER BY dv.latest_created_at DESC` + rows, err := conn.Query(ctx, query, args) if err != nil { - return MavenBuildListResponse{}, err + return MavenVersionsResponse{}, err + } + + type buildQueryResult struct { + GroupID string + ArtifactID string + Version string + BuildsJSON []byte } - artifacts, err := pgx.CollectRows(rows, pgx.RowToStructByName[mavenArtifactQueryResult]) + queryResults, err := pgx.CollectRows(rows, pgx.RowToStructByName[buildQueryResult]) if err != nil { - return MavenBuildListResponse{}, err - } - - // Convert to response format - results := make([]MavenBuildListItem, len(artifacts)) - for i, artifact := range artifacts { - results[i] = MavenBuildListItem{ - GroupID: artifact.GroupID, - ArtifactID: artifact.ArtifactID, - Version: artifact.Version, - Release: extractRelease(artifact.Filename), - Filename: artifact.Filename, - CreatedAt: artifact.CreatedAt.Format(time.RFC3339), + return MavenVersionsResponse{}, err + } + + results := make([]MavenVersionsItem, 0, len(queryResults)) + for _, qr := range queryResults { + var builds []struct { + Version string `json:"version"` + Filename string `json:"filename"` + CreatedAt time.Time `json:"created_at"` + } + + if err := json.Unmarshal(qr.BuildsJSON, &builds); err != nil { + return MavenVersionsResponse{}, fmt.Errorf("failed to parse builds: %w", err) } + + buildInfos := make([]MavenBuildInfo, 0, len(builds)) + for _, b := range builds { + buildInfos = append(buildInfos, MavenBuildInfo{ + Version: b.Version, + Release: extractRelease(b.Filename), + Filename: b.Filename, + CreatedAt: b.CreatedAt.Format(time.RFC3339), + }) + } + + results = append(results, MavenVersionsItem{ + GroupID: qr.GroupID, + ArtifactID: qr.ArtifactID, + Version: qr.Version, + Builds: buildInfos, + }) } - return MavenBuildListResponse{ + return MavenVersionsResponse{ Results: results, Total: countTotal, Limit: pageOpts.Limit, diff --git a/pkg/tangy/maven_test.go b/pkg/tangy/maven_test.go index 3bf5be3..fe4000c 100644 --- a/pkg/tangy/maven_test.go +++ b/pkg/tangy/maven_test.go @@ -224,7 +224,7 @@ func TestMockTangyMavenPackageList(t *testing.T) { assert.Equal(t, expected, got) } -func TestMockTangyMavenBuildList(t *testing.T) { +func TestMockTangyMavenVersionsList(t *testing.T) { t.Parallel() mockTangy := NewMockTangy(t) @@ -232,15 +232,20 @@ func TestMockTangyMavenBuildList(t *testing.T) { repoHref := "/api/pulp/default/api/v3/repositories/maven/maven/018c1c95-4281-76eb-b277-842cbad524f4/" pageOpts := PageOptions{Offset: 0, Limit: 10} - expected := MavenBuildListResponse{ - Results: []MavenBuildListItem{ + expected := MavenVersionsResponse{ + Results: []MavenVersionsItem{ { GroupID: "junit", ArtifactID: "junit", Version: "4.13.2", - Release: "", - Filename: "junit-4.13.2.pom", - CreatedAt: "2024-01-01T12:00:00Z", + Builds: []MavenBuildInfo{ + { + Version: "4.13.2", + Release: "", + Filename: "junit-4.13.2.pom", + CreatedAt: "2024-01-01T12:00:00Z", + }, + }, }, }, Total: 1, @@ -248,9 +253,9 @@ func TestMockTangyMavenBuildList(t *testing.T) { Offset: 0, } - mockTangy.On("MavenBuildList", ctx, repoHref, "junit", "junit", "4.13.2", pageOpts).Return(expected, nil) + mockTangy.On("MavenVersionsList", ctx, repoHref, "junit", "junit", "4.13.2", pageOpts).Return(expected, nil) - got, err := mockTangy.MavenBuildList(ctx, repoHref, "junit", "junit", "4.13.2", pageOpts) + got, err := mockTangy.MavenVersionsList(ctx, repoHref, "junit", "junit", "4.13.2", pageOpts) require.NoError(t, err) assert.Equal(t, expected, got) } diff --git a/pkg/tangy/tangy_mock.go b/pkg/tangy/tangy_mock.go index 3917c4d..86ad69f 100644 --- a/pkg/tangy/tangy_mock.go +++ b/pkg/tangy/tangy_mock.go @@ -70,23 +70,23 @@ func (_c *MockTangy_Close_Call) RunAndReturn(run func()) *MockTangy_Close_Call { return _c } -// MavenBuildList provides a mock function for the type MockTangy -func (_mock *MockTangy) MavenBuildList(ctx context.Context, repositoryHref string, groupID string, artifactID string, version string, pageOpts PageOptions) (MavenBuildListResponse, error) { +// MavenVersionsList provides a mock function for the type MockTangy +func (_mock *MockTangy) MavenVersionsList(ctx context.Context, repositoryHref string, groupID string, artifactID string, version string, pageOpts PageOptions) (MavenVersionsResponse, error) { ret := _mock.Called(ctx, repositoryHref, groupID, artifactID, version, pageOpts) if len(ret) == 0 { - panic("no return value specified for MavenBuildList") + panic("no return value specified for MavenVersionsList") } - var r0 MavenBuildListResponse + var r0 MavenVersionsResponse var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, string, string, PageOptions) (MavenBuildListResponse, error)); ok { + if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, string, string, PageOptions) (MavenVersionsResponse, error)); ok { return returnFunc(ctx, repositoryHref, groupID, artifactID, version, pageOpts) } - if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, string, string, PageOptions) MavenBuildListResponse); ok { + if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, string, string, PageOptions) MavenVersionsResponse); ok { r0 = returnFunc(ctx, repositoryHref, groupID, artifactID, version, pageOpts) } else { - r0 = ret.Get(0).(MavenBuildListResponse) + r0 = ret.Get(0).(MavenVersionsResponse) } if returnFunc, ok := ret.Get(1).(func(context.Context, string, string, string, string, PageOptions) error); ok { r1 = returnFunc(ctx, repositoryHref, groupID, artifactID, version, pageOpts) @@ -96,23 +96,23 @@ func (_mock *MockTangy) MavenBuildList(ctx context.Context, repositoryHref strin return r0, r1 } -// MockTangy_MavenBuildList_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MavenBuildList' -type MockTangy_MavenBuildList_Call struct { +// MockTangy_MavenVersionsList_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MavenVersionsList' +type MockTangy_MavenVersionsList_Call struct { *mock.Call } -// MavenBuildList is a helper method to define mock.On call +// MavenVersionsList is a helper method to define mock.On call // - ctx context.Context // - repositoryHref string // - groupID string // - artifactID string // - version string // - pageOpts PageOptions -func (_e *MockTangy_Expecter) MavenBuildList(ctx any, repositoryHref any, groupID any, artifactID any, version any, pageOpts any) *MockTangy_MavenBuildList_Call { - return &MockTangy_MavenBuildList_Call{Call: _e.mock.On("MavenBuildList", ctx, repositoryHref, groupID, artifactID, version, pageOpts)} +func (_e *MockTangy_Expecter) MavenVersionsList(ctx any, repositoryHref any, groupID any, artifactID any, version any, pageOpts any) *MockTangy_MavenVersionsList_Call { + return &MockTangy_MavenVersionsList_Call{Call: _e.mock.On("MavenVersionsList", ctx, repositoryHref, groupID, artifactID, version, pageOpts)} } -func (_c *MockTangy_MavenBuildList_Call) Run(run func(ctx context.Context, repositoryHref string, groupID string, artifactID string, version string, pageOpts PageOptions)) *MockTangy_MavenBuildList_Call { +func (_c *MockTangy_MavenVersionsList_Call) Run(run func(ctx context.Context, repositoryHref string, groupID string, artifactID string, version string, pageOpts PageOptions)) *MockTangy_MavenVersionsList_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -150,12 +150,12 @@ func (_c *MockTangy_MavenBuildList_Call) Run(run func(ctx context.Context, repos return _c } -func (_c *MockTangy_MavenBuildList_Call) Return(mavenBuildListResponse MavenBuildListResponse, err error) *MockTangy_MavenBuildList_Call { - _c.Call.Return(mavenBuildListResponse, err) +func (_c *MockTangy_MavenVersionsList_Call) Return(mavenVersionsResponse MavenVersionsResponse, err error) *MockTangy_MavenVersionsList_Call { + _c.Call.Return(mavenVersionsResponse, err) return _c } -func (_c *MockTangy_MavenBuildList_Call) RunAndReturn(run func(ctx context.Context, repositoryHref string, groupID string, artifactID string, version string, pageOpts PageOptions) (MavenBuildListResponse, error)) *MockTangy_MavenBuildList_Call { +func (_c *MockTangy_MavenVersionsList_Call) RunAndReturn(run func(ctx context.Context, repositoryHref string, groupID string, artifactID string, version string, pageOpts PageOptions) (MavenVersionsResponse, error)) *MockTangy_MavenVersionsList_Call { _c.Call.Return(run) return _c }