From 5d08acf8997745b745efbc3d5ff8bc86d4196a11 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 5 Jan 2026 11:23:00 -0800 Subject: [PATCH 1/8] setup --- src/server/lib/__tests__/utils.test.ts | 8 ++++++++ src/server/lib/utils.ts | 9 +++++++++ src/server/services/globalConfig.ts | 1 + src/server/services/types/globalConfig.ts | 11 +++++++++++ src/shared/config.ts | 2 ++ src/shared/constants.ts | 5 +++++ 6 files changed, 36 insertions(+) diff --git a/src/server/lib/__tests__/utils.test.ts b/src/server/lib/__tests__/utils.test.ts index cc934628..2abb59f8 100644 --- a/src/server/lib/__tests__/utils.test.ts +++ b/src/server/lib/__tests__/utils.test.ts @@ -45,6 +45,7 @@ jest.mock('server/services/globalConfig', () => { getLabels: jest.fn().mockResolvedValue({ deploy: ['lifecycle-deploy!', 'custom-deploy!'], disabled: ['lifecycle-disabled!', 'no-deploy!'], + keep: ['lifecycle-keep!'], statusComments: ['lifecycle-status-comments!', 'show-status!'], defaultStatusComments: true, defaultControlComments: true, @@ -355,6 +356,7 @@ describe('hasDeployLabel', () => { const mockService = GlobalConfigService.getInstance() as jest.Mocked; mockService.getLabels.mockResolvedValueOnce({ disabled: ['lifecycle-disabled!'], + keep: ['lifecycle-keep!'], statusComments: ['lifecycle-status-comments!'], defaultStatusComments: true, } as any); @@ -367,6 +369,7 @@ describe('hasDeployLabel', () => { mockService.getLabels.mockResolvedValueOnce({ deploy: [], disabled: ['lifecycle-disabled!'], + keep: ['lifecycle-keep!'], statusComments: ['lifecycle-status-comments!'], defaultStatusComments: true, defaultControlComments: true, @@ -433,6 +436,7 @@ describe('getDeployLabel', () => { const mockService = GlobalConfigService.getInstance() as jest.Mocked; mockService.getLabels.mockResolvedValueOnce({ disabled: ['lifecycle-disabled!'], + keep: ['lifecycle-keep!'], statusComments: ['lifecycle-status-comments!'], defaultStatusComments: true, } as any); @@ -445,6 +449,7 @@ describe('getDeployLabel', () => { mockService.getLabels.mockResolvedValueOnce({ deploy: [], disabled: ['lifecycle-disabled!'], + keep: ['lifecycle-keep!'], statusComments: ['lifecycle-status-comments!'], defaultStatusComments: true, defaultControlComments: true, @@ -494,6 +499,7 @@ describe('isDefaultStatusCommentsEnabled', () => { mockService.getLabels.mockResolvedValueOnce({ deploy: ['lifecycle-deploy!'], disabled: ['lifecycle-disabled!'], + keep: ['lifecycle-keep!'], statusComments: ['lifecycle-status-comments!'], } as any); const result = await isDefaultStatusCommentsEnabled(); @@ -519,6 +525,7 @@ describe('isControlCommentsEnabled', () => { mockService.getLabels.mockResolvedValueOnce({ deploy: ['lifecycle-deploy!'], disabled: ['lifecycle-disabled!'], + keep: ['lifecycle-keep!'], statusComments: ['lifecycle-status-comments!'], defaultStatusComments: true, } as any); @@ -532,6 +539,7 @@ describe('isControlCommentsEnabled', () => { mockService.getLabels.mockResolvedValueOnce({ deploy: ['lifecycle-deploy!'], disabled: ['lifecycle-disabled!'], + keep: ['lifecycle-keep!'], statusComments: ['lifecycle-status-comments!'], defaultStatusComments: true, defaultControlComments: false, diff --git a/src/server/lib/utils.ts b/src/server/lib/utils.ts index e69e9594..afde9839 100644 --- a/src/server/lib/utils.ts +++ b/src/server/lib/utils.ts @@ -257,6 +257,15 @@ export const getDisabledLabel = async (): Promise => { return labelsConfig?.disabled?.[0] || FallbackLabels.DISABLED; }; +/** + * Get the first keep label from configuration for user-facing messages + * @returns Promise First keep label from config + */ +export const getKeepLabel = async (): Promise => { + const labelsConfig = await GlobalConfigService.getInstance().getLabels(); + return labelsConfig?.keep?.[0] || FallbackLabels.KEEP; +}; + /** * Get the first status comment label from configuration for user-facing messages * @returns Promise First status comment label from config diff --git a/src/server/services/globalConfig.ts b/src/server/services/globalConfig.ts index ef9e7d8e..5140bd2e 100644 --- a/src/server/services/globalConfig.ts +++ b/src/server/services/globalConfig.ts @@ -125,6 +125,7 @@ export default class GlobalConfigService extends BaseService { return { deploy: ['lifecycle-deploy!'], disabled: ['lifecycle-disabled!'], + keep: ['lifecycle-keep!'], statusComments: ['lifecycle-status-comments!'], defaultStatusComments: true, defaultControlComments: true, diff --git a/src/server/services/types/globalConfig.ts b/src/server/services/types/globalConfig.ts index 6fb5a318..b4efb01b 100644 --- a/src/server/services/types/globalConfig.ts +++ b/src/server/services/types/globalConfig.ts @@ -37,6 +37,7 @@ export type GlobalConfig = { features: Record; app_setup: AppSetup; labels: LabelsConfig; + ttl_cleanup?: TTLCleanupConfig; }; export type AppSetup = { @@ -144,7 +145,17 @@ export type ResourceRequirements = { export type LabelsConfig = { deploy: string[]; disabled: string[]; + keep: string[]; statusComments: string[]; defaultStatusComments: boolean; defaultControlComments: boolean; }; + +export type TTLCleanupConfig = { + enabled: boolean; + dryRun: boolean; + inactivityDays: number; + checkIntervalMinutes: number; + commentTemplate?: string; + excludedRepositories: string[]; +}; diff --git a/src/shared/config.ts b/src/shared/config.ts index 8a3ec1c0..9f03c430 100644 --- a/src/shared/config.ts +++ b/src/shared/config.ts @@ -107,6 +107,8 @@ export const QUEUE_NAMES = { WEBHOOK_PROCESSING: `webhook_processing_${JOB_VERSION}`, COMMENT_QUEUE: `comment_queue_${JOB_VERSION}`, CLEANUP: `cleanup_${JOB_VERSION}`, + // NOTE: No version suffix - singleton queue shared across app instances + TTL_CLEANUP: 'ttl_cleanup', GLOBAL_CONFIG_CACHE_REFRESH: 'global_config_cache_refresh', GITHUB_CLIENT_TOKEN_CACHE_REFRESH: 'github_client_token_cache_refresh', INGRESS_MANIFEST: `ingress_manifest_${JOB_VERSION}`, diff --git a/src/shared/constants.ts b/src/shared/constants.ts index edf0d195..eb014e11 100644 --- a/src/shared/constants.ts +++ b/src/shared/constants.ts @@ -110,6 +110,10 @@ export enum GithubPullRequestActions { SYNCHRONIZE = 'synchronize', } +// TTL Cleanup defaults +export const DEFAULT_TTL_INACTIVITY_DAYS = 14; +export const DEFAULT_TTL_CHECK_INTERVAL_MINUTES = 240; // 4 hours + export enum CommentParser { HEADER = `----EDIT BELOW THIS LINE----`, FOOTER = `----EDIT ABOVE THIS LINE----`, @@ -118,6 +122,7 @@ export enum CommentParser { export enum FallbackLabels { DEPLOY = 'lifecycle-deploy!', DISABLED = 'lifecycle-disabled!', + KEEP = 'lifecycle-keep!', DEPLOY_STG = 'lifecycle-stg-deploy!', STATUS_COMMENTS = 'lifecycle-status-comments!', } From d49af4a07475af8d2d915b0c648f3e8ec3b9f2c4 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 5 Jan 2026 11:47:02 -0800 Subject: [PATCH 2/8] create ttl config --- .../db/migrations/006_add_ttl_cleanup.ts | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/server/db/migrations/006_add_ttl_cleanup.ts diff --git a/src/server/db/migrations/006_add_ttl_cleanup.ts b/src/server/db/migrations/006_add_ttl_cleanup.ts new file mode 100644 index 00000000..b3ecc8fa --- /dev/null +++ b/src/server/db/migrations/006_add_ttl_cleanup.ts @@ -0,0 +1,81 @@ +/** + * Copyright 2025 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + const existingLabels = await knex('global_config').where('key', 'labels').first(); + + const defaultLabelsConfig = { + deploy: ['lifecycle-deploy!'], + disabled: ['lifecycle-disabled!'], + keep: ['lifecycle-keep!'], + statusComments: ['lifecycle-status-comments!'], + defaultStatusComments: true, + defaultControlComments: true, + }; + + if (existingLabels) { + const mergedConfig = { ...defaultLabelsConfig, ...existingLabels.config }; + + await knex('global_config').where('key', 'labels').update({ + config: mergedConfig, + updatedAt: knex.fn.now(), + }); + } else { + await knex('global_config').insert({ + key: 'labels', + config: defaultLabelsConfig, + createdAt: knex.fn.now(), + updatedAt: knex.fn.now(), + deletedAt: null, + description: 'Configurable PR labels for deploy, disabled, keep, and status comments', + }); + } + + await knex('global_config').insert({ + key: 'ttl_cleanup', + config: { + enabled: false, + dryRun: true, + inactivityDays: 14, + checkIntervalMinutes: 240, + commentTemplate: + 'This environment has been inactive for {inactivityDays} days and will be automatically cleaned up. Add the {keepLabel} label to prevent cleanup.', + excludedRepositories: [], + }, + createdAt: knex.fn.now(), + updatedAt: knex.fn.now(), + deletedAt: null, + description: + 'TTL-based automatic cleanup configuration for inactive PR environments. Set enabled to true to activate cleanup (starts in dryRun mode). Environments with the keep label are excluded.', + }); +} + +export async function down(knex: Knex): Promise { + const existingLabels = await knex('global_config').where('key', 'labels').first(); + + if (existingLabels && existingLabels.config?.keep) { + const { keep: _keep, ...restConfig } = existingLabels.config; + + await knex('global_config').where('key', 'labels').update({ + config: restConfig, + updatedAt: knex.fn.now(), + }); + } + + await knex('global_config').where('key', 'ttl_cleanup').delete(); +} From 870ebf0b1eebd4cd5a6cbb84d41dd70ae0a2d4c8 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 5 Jan 2026 11:55:55 -0800 Subject: [PATCH 3/8] add github helper method --- src/server/lib/github/index.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/server/lib/github/index.ts b/src/server/lib/github/index.ts index d9e6da8c..21e6f006 100644 --- a/src/server/lib/github/index.ts +++ b/src/server/lib/github/index.ts @@ -113,6 +113,32 @@ export async function getPullRequestByRepositoryFullName( } } +/** + * Fetches current labels from GitHub API for a pull request + * Used by TTL cleanup to avoid stale label data from database + */ +export async function getPullRequestLabels({ + installationId, + pullRequestNumber, + fullName, +}: { + installationId: number; + pullRequestNumber: number; + fullName: string; +}): Promise { + try { + const client = await createOctokitClient({ + installationId, + caller: 'getPullRequestLabels', + }); + const response = await client.request(`GET /repos/${fullName}/issues/${pullRequestNumber}`); + return response.data.labels.map((label: any) => label.name); + } catch (error) { + initialLogger.error(`[GITHUB ${fullName}/${pullRequestNumber}] Unable to fetch labels: ${error}`); + throw error; + } +} + export async function createDeploy({ owner, name, branch, installationId, logger = initialLogger }: RepoOptions) { try { const octokit = await createOctokitClient({ installationId, caller: 'createDeploy' }); From dee966138cd8c925cc0f6fe188e250cc672fdd50 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 5 Jan 2026 12:01:49 -0800 Subject: [PATCH 4/8] create/patch ns labels helpers --- src/server/lib/kubernetes.ts | 194 ++++++++++++++++++++++++++--------- 1 file changed, 147 insertions(+), 47 deletions(-) diff --git a/src/server/lib/kubernetes.ts b/src/server/lib/kubernetes.ts index 47ededd6..b4d4312a 100644 --- a/src/server/lib/kubernetes.ts +++ b/src/server/lib/kubernetes.ts @@ -18,7 +18,7 @@ import rootLogger from './logger'; import yaml from 'js-yaml'; import _ from 'lodash'; import { Build, Deploy, Deployable, Service } from 'server/models'; -import { CLIDeployTypes, KubernetesDeployTypes, MEDIUM_TYPE } from 'shared/constants'; +import { CLIDeployTypes, KubernetesDeployTypes, MEDIUM_TYPE, DEFAULT_TTL_INACTIVITY_DAYS } from 'shared/constants'; import { shellPromise } from './shell'; import { flattenObject, waitUntil } from 'server/lib/utils'; import { ServiceDiskConfig } from 'server/models/yaml'; @@ -56,6 +56,127 @@ async function namespaceExists(client: k8s.CoreV1Api, name: string): Promise { + let daysToExpire = DEFAULT_TTL_INACTIVITY_DAYS; + try { + const globalConfig = await GlobalConfigService.getInstance().getAllConfigs(); + daysToExpire = globalConfig.ttl_cleanup?.inactivityDays ?? DEFAULT_TTL_INACTIVITY_DAYS; + } catch (error) { + logger.warn( + `[NS ${buildUUID}] Failed to get TTL config, using default ${DEFAULT_TTL_INACTIVITY_DAYS} days: ${error}` + ); + } + return { daysToExpire }; +} + +/** + * Generates TTL-related labels for namespace creation + */ +async function generateTTLLabels({ + uuid, + staticEnv, + ttl, + buildUUID, +}: { + uuid: string; + staticEnv: boolean; + ttl: boolean; + buildUUID: string; +}): Promise<{ labels: Record; logMessage: string }> { + const baseLabels = { 'lfc/uuid': uuid }; + + // Static or TTL disabled - only set enable flag + if (staticEnv || !ttl) { + const reason = staticEnv ? 'static env' : 'lifecycle-keep! label present'; + return { + labels: { + ...baseLabels, + 'lfc/ttl-enable': 'false', + }, + logMessage: `with TTL disabled (${reason})`, + }; + } + + // TTL enabled - set enable flag + expiration timestamps + const { daysToExpire } = await getTTLConfig(buildUUID); + const timeToExpire = Date.now() + daysToExpire * 24 * 60 * 60 * 1000; + + return { + labels: { + ...baseLabels, + 'lfc/ttl-enable': 'true', + 'lfc/ttl-createdAtUnix': Date.now().toString(), + 'lfc/ttl-createdAt': new Date().toISOString().split('T')[0], + 'lfc/ttl-expireAtUnix': timeToExpire.toString(), + 'lfc/ttl-expireAt': new Date(timeToExpire).toISOString().split('T')[0], + }, + logMessage: `with TTL enabled (${daysToExpire} day expiration)`, + }; +} + +/** + * Generates patch operations for updating TTL labels on existing namespace + */ +async function generateTTLPatch({ + ttl, + buildUUID, +}: { + ttl: boolean; + buildUUID: string; +}): Promise<{ patch: any[]; logMessage: string }> { + // TTL disabled - only update enable flag + if (!ttl) { + return { + patch: [ + { + op: 'add', + path: '/metadata/labels/lfc~1ttl-enable', + value: 'false', + }, + ], + logMessage: 'to disable TTL (lifecycle-keep! present)', + }; + } + + // TTL enabled - update enable flag + all expiration timestamps + const { daysToExpire } = await getTTLConfig(buildUUID); + const timeToExpire = Date.now() + daysToExpire * 24 * 60 * 60 * 1000; + + return { + patch: [ + { + op: 'add', + path: '/metadata/labels/lfc~1ttl-enable', + value: 'true', + }, + { + op: 'add', + path: '/metadata/labels/lfc~1ttl-createdAtUnix', + value: Date.now().toString(), + }, + { + op: 'add', + path: '/metadata/labels/lfc~1ttl-createdAt', + value: new Date().toISOString().split('T')[0], + }, + { + op: 'add', + path: '/metadata/labels/lfc~1ttl-expireAtUnix', + value: timeToExpire.toString(), + }, + { + op: 'add', + path: '/metadata/labels/lfc~1ttl-expireAt', + value: new Date(timeToExpire).toISOString().split('T')[0], + }, + ], + logMessage: `with new TTL expiration (${daysToExpire} days)`, + }; +} + /** * */ @@ -63,36 +184,29 @@ export async function createOrUpdateNamespace({ name, buildUUID, staticEnv, + ttl = true, }: { name: string; buildUUID: string; staticEnv: boolean; + ttl?: boolean; }) { const kc = new k8s.KubeConfig(); kc.loadFromDefault(); const client = kc.makeApiClient(k8s.CoreV1Api); - const daysToExpire = 4; - let labels: Record = {}; - const timeToExpire = Date.now() + daysToExpire * 24 * 60 * 60 * 1000; - const lc_uuid = name.replace('env-', ''); + const uuid = name.replace('env-', ''); + + // Generate TTL labels using helper function + const { labels, logMessage } = await generateTTLLabels({ + uuid, + staticEnv, + ttl, + buildUUID, + }); + + logger.info(`[NS ${buildUUID}] Creating/updating namespace ${name} ${logMessage}`); - if (!staticEnv) { - labels = { - lc_uuid, - 'lfc/ttl-enable': 'true', - 'lfc/ttl-createdAtUnix': Date.now().toString(), - 'lfc/ttl-createdAt': new Date().toISOString().split('T')[0], - 'lfc/ttl-expireAtUnix': timeToExpire.toString(), - 'lfc/ttl-expireAt': new Date(timeToExpire).toISOString().split('T')[0], - }; - } else { - labels = { - lc_uuid, - ttl_enable: 'false', - }; - } - logger.info(`[NS ${buildUUID}] Creating namespace ${name} with ttl_enable: ${labels.ttl_enable}`); const namespace = { apiVersion: 'v1', kind: 'Namespace', @@ -101,45 +215,31 @@ export async function createOrUpdateNamespace({ labels, }, }; + if (await namespaceExists(client, name)) { + // Only update TTL labels if not static env if (!staticEnv) { - const patch = [ - { - op: 'add', - path: '/metadata/labels/lfc~1ttl-createdAtUnix', - value: Date.now().toString(), - }, - { - op: 'add', - path: '/metadata/labels/lfc~1ttl-createdAt', - value: new Date().toISOString().split('T')[0], - }, - { - op: 'add', - path: '/metadata/labels/lfc~1ttl-expireAtUnix', - value: timeToExpire.toString(), - }, - { - op: 'add', - path: '/metadata/labels/lfc~1ttl-expireAt', - value: new Date(timeToExpire).toISOString().split('T')[0], - }, - ]; + const { patch, logMessage: patchMessage } = await generateTTLPatch({ + ttl, + buildUUID, + }); + await client.patchNamespace(name, patch, undefined, undefined, undefined, undefined, undefined, { headers: { 'Content-Type': 'application/json-patch+json' }, }); - logger.info(`[NS ${buildUUID}] The namespace has been successfully updated`); + logger.info(`[NS ${buildUUID}] Updated namespace ${patchMessage}`); return; } else { - logger.info(`[NS ${buildUUID}] The namespace already exists and is static env, skipping update labels`); + logger.info(`[NS ${buildUUID}] Namespace exists and is static env, skipping update`); return; } } + try { await client.createNamespace(namespace); - logger.debug(`[NS ${name}] Namespace created `); + logger.debug(`[NS ${name}] Namespace created`); } catch (err) { - logger.error(`[NS ${name}]Error creating namespace: ${err}`); + logger.error(`[NS ${name}] Error creating namespace: ${err}`); throw err; } } From fd83e40f6b729a3c9ee8987d3362e7ae17c5f20c Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 5 Jan 2026 13:26:52 -0800 Subject: [PATCH 5/8] create ttl cleanup queue and service --- src/server/services/index.ts | 2 + src/server/services/ttlCleanup.ts | 443 +++++++++++++++++++++++++++++ src/server/services/types/index.ts | 2 + 3 files changed, 447 insertions(+) create mode 100644 src/server/services/ttlCleanup.ts diff --git a/src/server/services/index.ts b/src/server/services/index.ts index f1b5719f..e6a98716 100644 --- a/src/server/services/index.ts +++ b/src/server/services/index.ts @@ -29,6 +29,7 @@ import Deployable from 'server/services/deployable'; import BotUser from 'server/services/botUser'; import GlobalConfig from 'server/services/globalConfig'; import LabelService from 'server/services/label'; +import TTLCleanupService from 'server/services/ttlCleanup'; import { IServices } from 'server/services/types'; export default function createAndBindServices(): IServices { @@ -48,5 +49,6 @@ export default function createAndBindServices(): IServices { BotUser: new BotUser(), GlobalConfig: GlobalConfig.getInstance(), LabelService: new LabelService(), + TTLCleanupService: new TTLCleanupService(), }; } diff --git a/src/server/services/ttlCleanup.ts b/src/server/services/ttlCleanup.ts new file mode 100644 index 00000000..046804c3 --- /dev/null +++ b/src/server/services/ttlCleanup.ts @@ -0,0 +1,443 @@ +/** + * Copyright 2025 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Service from './_service'; +import { Queue, Job } from 'bullmq'; +import { QUEUE_NAMES } from 'shared/config'; +import { redisClient } from 'server/lib/dependencies'; +import rootLogger from 'server/lib/logger'; +import * as k8s from '@kubernetes/client-node'; +import { updatePullRequestLabels, createOrUpdatePullRequestComment, getPullRequestLabels } from 'server/lib/github'; +import { getKeepLabel, getDisabledLabel, getDeployLabel } from 'server/lib/utils'; +import { Build, PullRequest } from 'server/models'; +import Metrics from 'server/lib/metrics'; +import { DEFAULT_TTL_INACTIVITY_DAYS, DEFAULT_TTL_CHECK_INTERVAL_MINUTES } from 'shared/constants'; +import GlobalConfigService from './globalConfig'; + +const logger = rootLogger.child({ + filename: 'services/ttlCleanup.ts', +}); + +interface TTLCleanupJob { + dryRun?: boolean; +} + +interface StaleEnvironment { + namespace: string; + buildUUID: string; + build: Build; + pullRequest: PullRequest; + daysExpired: number; + currentLabels: string[]; + hadLabelDrift: boolean; +} + +export default class TTLCleanupService extends Service { + /** + * Queue for managing TTL cleanup operations + */ + ttlCleanupQueue: Queue = this.queueManager.registerQueue(QUEUE_NAMES.TTL_CLEANUP, { + connection: redisClient.getConnection(), + defaultJobOptions: { + attempts: 1, + removeOnComplete: true, + removeOnFail: false, + }, + }); + + /** + * Process TTL cleanup queue jobs + */ + processTTLCleanupQueue = async (job: Job) => { + try { + // Always read fresh config to handle runtime config changes + const config = await this.getTTLConfig(); + + if (!config.enabled) { + logger.info('[TTL] TTL cleanup is disabled, skipping'); + return; + } + + // Job data takes precedence (for manual API calls), fall back to config for scheduled jobs + const dryRun = job.data.dryRun ?? config.dryRun; + const source = job.data.dryRun !== undefined ? 'api-override' : 'config'; + + logger.info('[TTL] Starting TTL cleanup job', { + dryRun, + source, + jobDataDryRun: job.data.dryRun, + configDryRun: config.dryRun, + }); + + const staleEnvironments = await this.findStaleEnvironments(config.inactivityDays, config.excludedRepositories); + + logger.info(`[TTL] Found ${staleEnvironments.length} stale environments`, { + inactivityDays: config.inactivityDays, + dryRun, + }); + + let successCount = 0; + let errorCount = 0; + + for (const env of staleEnvironments) { + try { + if (dryRun) { + const dbLabels = this.parseLabels(env.pullRequest.labels); + + logger.info(`[TTL ${env.buildUUID}] [DRY RUN] Would clean up environment (NO ACTION TAKEN)`, { + namespace: env.namespace, + prNumber: env.pullRequest.pullRequestNumber, + fullName: env.pullRequest.fullName, + daysExpired: env.daysExpired, + currentLabelsFromGitHub: env.currentLabels, + labelsInDatabase: dbLabels, + labelDriftDetected: env.hadLabelDrift, + }); + successCount++; + } else { + logger.info(`[TTL ${env.buildUUID}] Cleaning up stale environment`, { + namespace: env.namespace, + prNumber: env.pullRequest.pullRequestNumber, + fullName: env.pullRequest.fullName, + }); + await this.cleanupStaleEnvironment(env, config.inactivityDays, config.commentTemplate, dryRun); + successCount++; + } + } catch (error) { + errorCount++; + logger.error(`[TTL ${env.buildUUID}] Failed to cleanup environment`, { + namespace: env.namespace, + error, + }); + } + } + + logger.info('[TTL] TTL cleanup job completed', { + totalFound: staleEnvironments.length, + successCount, + errorCount, + dryRun, + }); + } catch (error) { + logger.error('[TTL] Error in TTL cleanup job', { error }); + throw error; + } + }; + + private parseLabels(labels: string | string[] | null): string[] { + if (!labels) return []; + return typeof labels === 'string' ? JSON.parse(labels) : labels; + } + + private async getTTLConfig() { + const globalConfig = await GlobalConfigService.getInstance().getAllConfigs(); + + return { + enabled: globalConfig.ttl_cleanup?.enabled ?? false, + dryRun: globalConfig.ttl_cleanup?.dryRun ?? false, + inactivityDays: globalConfig.ttl_cleanup?.inactivityDays ?? DEFAULT_TTL_INACTIVITY_DAYS, + checkIntervalMinutes: globalConfig.ttl_cleanup?.checkIntervalMinutes ?? DEFAULT_TTL_CHECK_INTERVAL_MINUTES, + commentTemplate: globalConfig.ttl_cleanup?.commentTemplate ?? this.getDefaultCommentTemplate(), + excludedRepositories: globalConfig.ttl_cleanup?.excludedRepositories ?? [], + }; + } + + /** + * Find stale environments that haven't had activity within the TTL window + */ + private async findStaleEnvironments( + _inactivityDays: number, + excludedRepositories: string[] + ): Promise { + const staleEnvironments: StaleEnvironment[] = []; + const now = Date.now(); + + try { + const kc = new k8s.KubeConfig(); + kc.loadFromDefault(); + const client = kc.makeApiClient(k8s.CoreV1Api); + + const namespacesResponse = await client.listNamespace( + undefined, + undefined, + undefined, + undefined, + 'lfc/ttl-enable=true' + ); + + const namespaces = namespacesResponse.body.items; + + logger.info(`[TTL] Scanning ${namespaces.length} namespaces with TTL enabled`); + + // Fetch dynamic labels once at the start + const keepLabel = await getKeepLabel(); + const disabledLabel = await getDisabledLabel(); + + for (const namespace of namespaces) { + const nsName = namespace.metadata?.name; + const labels = namespace.metadata?.labels || {}; + + if (!nsName || !nsName.startsWith('env-')) { + continue; + } + + const expireAtUnix = labels['lfc/ttl-expireAtUnix']; + + if (!expireAtUnix) { + logger.debug(`[TTL] Namespace ${nsName} has no TTL expiration label, skipping`); + continue; + } + + const expireTime = parseInt(expireAtUnix, 10); + + if (isNaN(expireTime) || expireTime > now) { + continue; + } + + const daysExpired = Math.floor((now - expireTime) / (1000 * 60 * 60 * 24)); + + const buildUUID = labels['lfc/uuid']; // Use lfc/uuid (intentional difference) + if (!buildUUID) { + logger.warn(`[TTL] Namespace ${nsName} has no lfc/uuid label, skipping`); + continue; + } + + logger.debug(`[TTL ${buildUUID}] Namespace ${nsName} expired ${daysExpired} days ago`); + + const build = await this.db.models.Build.query() + .findOne({ uuid: buildUUID }) + .withGraphFetched('[pullRequest.repository]'); + + if (!build) { + logger.warn(`[TTL ${buildUUID}] No build found for namespace ${nsName}, skipping`); + continue; + } + + if (build.status === 'torn_down' || build.status === 'pending') { + logger.debug(`[TTL ${buildUUID}] Build is already ${build.status}, skipping`); + continue; + } + + if (build.isStatic) { + logger.debug(`[TTL ${buildUUID}] Build is static environment, skipping`); + continue; + } + + const pullRequest = build.pullRequest; + + if (!pullRequest) { + logger.warn(`[TTL ${buildUUID}] No pull request found, skipping`); + continue; + } + + if (pullRequest.status !== 'open') { + logger.debug(`[TTL ${buildUUID}] PR is ${pullRequest.status}, skipping`); + continue; + } + + if (excludedRepositories.length > 0 && excludedRepositories.includes(pullRequest.fullName)) { + logger.debug(`[TTL ${buildUUID}] Repository ${pullRequest.fullName} is excluded from TTL cleanup, skipping`); + continue; + } + + // Fetch current labels from GitHub to avoid stale data due to webhook incidents + let currentLabels: string[]; + try { + currentLabels = await getPullRequestLabels({ + installationId: pullRequest.repository.githubInstallationId, + pullRequestNumber: pullRequest.pullRequestNumber, + fullName: pullRequest.fullName, + }); + + logger.debug( + `[TTL ${buildUUID}] Fetched ${currentLabels.length} labels from GitHub: ${currentLabels.join(', ')}` + ); + + // Sync labels back to DB if they differ (self-healing) + const dbLabels = this.parseLabels(pullRequest.labels); + if (JSON.stringify(currentLabels.sort()) !== JSON.stringify(dbLabels.sort())) { + logger.info(`[TTL ${buildUUID}] Label drift detected, syncing to database`, { + dbLabels, + currentLabels, + }); + await pullRequest.$query().patch({ + labels: JSON.stringify(currentLabels) as any, + }); + } + } catch (error) { + logger.warn(`[TTL ${buildUUID}] Failed to fetch labels from GitHub, falling back to DB: ${error}`); + // Fallback to DB labels if GitHub API fails + currentLabels = this.parseLabels(pullRequest.labels); + } + + if (currentLabels.includes(keepLabel)) { + logger.debug(`[TTL ${buildUUID}] Has ${keepLabel} label (verified from GitHub), skipping`); + continue; + } + + if (currentLabels.includes(disabledLabel)) { + logger.debug(`[TTL ${buildUUID}] Already has ${disabledLabel} label (verified from GitHub), skipping`); + continue; + } + + // Store current labels and drift status for dry-run reporting + const dbLabels = this.parseLabels(pullRequest.labels); + const hadLabelDrift = JSON.stringify(currentLabels.sort()) !== JSON.stringify(dbLabels.sort()); + + staleEnvironments.push({ + namespace: nsName, + buildUUID, + build, + pullRequest, + daysExpired, + currentLabels, + hadLabelDrift, + }); + } + } catch (error) { + logger.error('[TTL] Error scanning K8s namespaces for stale environments', { error }); + throw error; + } + + return staleEnvironments; + } + + /** + * Cleanup a stale environment by updating labels and posting a comment + */ + private async cleanupStaleEnvironment( + env: StaleEnvironment, + inactivityDays: number, + commentTemplate: string | undefined, + dryRun: boolean + ) { + const { build, pullRequest, namespace } = env; + const buildUuid = build.uuid; + const repository = pullRequest.repository; + + logger.info(`[TTL ${buildUuid}] Cleaning up stale environment`, { + namespace, + prNumber: pullRequest.pullRequestNumber, + fullName: pullRequest.fullName, + daysExpired: env.daysExpired, + }); + + // Fetch dynamic labels at runtime + const deployLabel = await getDeployLabel(); + const disabledLabel = await getDisabledLabel(); + + const currentLabels = this.parseLabels(pullRequest.labels); + + const updatedLabels = currentLabels.filter((label) => label !== deployLabel).concat(disabledLabel); + + try { + await updatePullRequestLabels({ + installationId: repository.githubInstallationId, + pullRequestNumber: pullRequest.pullRequestNumber, + fullName: pullRequest.fullName, + labels: updatedLabels, + }); + + logger.info(`[TTL ${buildUuid}] Updated labels: removed ${deployLabel}, added ${disabledLabel}`, { + prNumber: pullRequest.pullRequestNumber, + }); + + const commentMessage = await this.generateCleanupComment(inactivityDays, commentTemplate); + + await createOrUpdatePullRequestComment({ + installationId: repository.githubInstallationId, + pullRequestNumber: pullRequest.pullRequestNumber, + fullName: pullRequest.fullName, + message: commentMessage, + commentId: null, + etag: null, + }); + + logger.info(`[TTL ${buildUuid}] Posted cleanup comment to PR`, { + prNumber: pullRequest.pullRequestNumber, + }); + + await pullRequest.$query().patch({ + labels: JSON.stringify(updatedLabels) as any, + }); + + // Track successful cleanup metric + const metrics = new Metrics('ttl.cleanup', { repositoryName: pullRequest.fullName }); + metrics.increment('total', { dry_run: dryRun.toString() }); + } catch (error) { + logger.error(`[TTL ${buildUuid}] Failed to cleanup stale environment`, { + namespace, + prNumber: pullRequest.pullRequestNumber, + error, + }); + throw error; + } + } + + /** + * Generate cleanup comment message with dynamic labels + */ + private async generateCleanupComment(inactivityDays: number, template?: string): Promise { + const defaultTemplate = this.getDefaultCommentTemplate(); + const commentTemplate = template || defaultTemplate; + + // Fetch dynamic labels for replacement + const keepLabel = await getKeepLabel(); + const disabledLabel = await getDisabledLabel(); + const deployLabel = await getDeployLabel(); + + return commentTemplate + .replace(/{inactivityDays}/g, inactivityDays.toString()) + .replace(/lifecycle-disabled!/g, disabledLabel) + .replace(/lifecycle-deploy!/g, deployLabel) + .replace(/lifecycle-keep!/g, keepLabel); + } + + /** + * Get default comment template with placeholders for dynamic labels + */ + private getDefaultCommentTemplate(): string { + return 'Tearing down lifecycle env since no activity in the past {inactivityDays} days.'; + } + + /** + * Setup TTL cleanup recurring job + */ + async setupTTLCleanupJob() { + const config = await this.getTTLConfig(); + + if (!config.enabled) { + logger.info('[TTL] TTL cleanup is disabled in global config'); + return; + } + + const intervalMs = config.checkIntervalMinutes * 60 * 1000; + + await this.ttlCleanupQueue.add( + 'ttl-cleanup', + {}, + { + repeat: { + every: intervalMs, + }, + } + ); + + logger.info( + `[TTL] TTL cleanup job scheduled every ${config.checkIntervalMinutes} minutes (${config.inactivityDays} day TTL, dryRun: ${config.dryRun})` + ); + } +} diff --git a/src/server/services/types/index.ts b/src/server/services/types/index.ts index a6b1c151..ba9b4b8e 100644 --- a/src/server/services/types/index.ts +++ b/src/server/services/types/index.ts @@ -29,6 +29,7 @@ import BotUser from 'server/services/botUser'; import GlobalConfig from 'server/services/globalConfig'; import GithubService from 'server/services/github'; import LabelService from 'server/services/label'; +import TTLCleanupService from 'server/services/ttlCleanup'; export interface IServices { BuildService: BuildService; @@ -46,6 +47,7 @@ export interface IServices { BotUser: BotUser; GlobalConfig: GlobalConfig; LabelService: LabelService; + TTLCleanupService: TTLCleanupService; } export * from 'server/services/types/github'; From 1ff08be66b10aa7078527a796c82b254b0029a1b Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 5 Jan 2026 13:32:48 -0800 Subject: [PATCH 6/8] register ttl queue, worker --- src/server/jobs/index.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/server/jobs/index.ts b/src/server/jobs/index.ts index 83c184f3..1089768f 100644 --- a/src/server/jobs/index.ts +++ b/src/server/jobs/index.ts @@ -72,6 +72,14 @@ export default function bootstrapJobs(services: IServices) { concurrency: 1, }); + /* Setup TTL cleanup job */ + services.TTLCleanupService.setupTTLCleanupJob(); + + queueManager.registerWorker(QUEUE_NAMES.TTL_CLEANUP, services.TTLCleanupService.processTTLCleanupQueue, { + connection: redisClient.getConnection(), + concurrency: 1, + }); + services.PullRequest.cleanupClosedPRQueue.add('cleanup', {}, {}); queueManager.registerWorker(QUEUE_NAMES.INGRESS_MANIFEST, services.Ingress.createOrUpdateIngressForBuild, { From 8132a4a6f55e0afff838c7279d351be0097a2481 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 5 Jan 2026 14:17:39 -0800 Subject: [PATCH 7/8] ttl api --- src/pages/api/v1/admin/ttl/cleanup.ts | 210 ++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 src/pages/api/v1/admin/ttl/cleanup.ts diff --git a/src/pages/api/v1/admin/ttl/cleanup.ts b/src/pages/api/v1/admin/ttl/cleanup.ts new file mode 100644 index 00000000..d159f652 --- /dev/null +++ b/src/pages/api/v1/admin/ttl/cleanup.ts @@ -0,0 +1,210 @@ +/** + * Copyright 2025 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NextApiRequest, NextApiResponse } from 'next'; +import rootLogger from 'server/lib/logger'; +import GlobalConfigService from 'server/services/globalConfig'; +import TTLCleanupService from 'server/services/ttlCleanup'; + +const logger = rootLogger.child({ + filename: 'v1/admin/ttl/cleanup.ts', +}); + +/** + * @openapi + * /api/v1/admin/ttl/cleanup: + * get: + * summary: Get TTL cleanup configuration + * description: Retrieves the current TTL cleanup configuration from global config + * tags: + * - Admin + * - TTL Cleanup + * responses: + * 200: + * description: Successfully retrieved TTL cleanup configuration + * content: + * application/json: + * schema: + * type: object + * properties: + * config: + * type: object + * properties: + * enabled: + * type: boolean + * description: Whether TTL cleanup is enabled + * dryRun: + * type: boolean + * description: Whether cleanup runs in dry-run mode + * inactivityDays: + * type: number + * description: Number of days of inactivity before cleanup + * checkIntervalMinutes: + * type: number + * description: How often cleanup job runs (in minutes) + * commentTemplate: + * type: string + * description: Template for PR comments + * excludedRepositories: + * type: array + * items: + * type: string + * description: List of repositories excluded from cleanup + * 405: + * description: Method not allowed + * content: + * application/json: + * schema: + * type: object + * properties: + * error: + * type: string + * example: DELETE is not allowed. + * 500: + * description: Internal server error + * content: + * application/json: + * schema: + * type: object + * properties: + * error: + * type: string + * example: Unable to retrieve TTL cleanup configuration + * post: + * summary: Manually trigger TTL cleanup + * description: Manually triggers a TTL cleanup job with optional configuration override + * tags: + * - Admin + * - TTL Cleanup + * requestBody: + * required: false + * content: + * application/json: + * schema: + * type: object + * properties: + * dryRun: + * type: boolean + * description: Override dry-run mode for this execution (optional) + * responses: + * 200: + * description: Successfully triggered TTL cleanup job + * content: + * application/json: + * schema: + * type: object + * properties: + * message: + * type: string + * example: TTL cleanup job triggered successfully + * jobId: + * type: string + * description: The ID of the queued job + * dryRun: + * type: boolean + * description: Whether this job will run in dry-run mode + * 400: + * description: Bad request - invalid parameters + * content: + * application/json: + * schema: + * type: object + * properties: + * error: + * type: string + * example: dryRun must be a boolean value + * 405: + * description: Method not allowed + * content: + * application/json: + * schema: + * type: object + * properties: + * error: + * type: string + * example: DELETE is not allowed. + * 500: + * description: Internal server error + * content: + * application/json: + * schema: + * type: object + * properties: + * error: + * type: string + * example: Unable to trigger TTL cleanup job + */ +// eslint-disable-next-line import/no-anonymous-default-export +export default async (req: NextApiRequest, res: NextApiResponse) => { + try { + switch (req.method) { + case 'GET': + return getTTLConfig(res); + case 'POST': + return triggerTTLCleanup(req, res); + default: + res.setHeader('Allow', ['GET', 'POST']); + return res.status(405).json({ error: `${req.method} is not allowed.` }); + } + } catch (error) { + logger.error(`Error occurred on TTL cleanup operation: \n ${error}`); + res.status(500).json({ error: 'An unexpected error occurred.' }); + } +}; + +async function getTTLConfig(res: NextApiResponse) { + try { + const configService = GlobalConfigService.getInstance(); + const globalConfig = await configService.getAllConfigs(); + const ttlConfig = globalConfig.ttl_cleanup; + + if (!ttlConfig) { + logger.warn('[API] TTL cleanup configuration not found in global config'); + return res.status(404).json({ error: 'TTL cleanup configuration not found' }); + } + + return res.status(200).json({ config: ttlConfig }); + } catch (error) { + logger.error(`[API] Error occurred retrieving TTL cleanup config: \n ${error}`); + return res.status(500).json({ error: 'Unable to retrieve TTL cleanup configuration' }); + } +} + +async function triggerTTLCleanup(req: NextApiRequest, res: NextApiResponse) { + try { + const { dryRun = false } = req.body || {}; + + // Validate dryRun parameter type + if (typeof dryRun !== 'boolean') { + return res.status(400).json({ error: 'dryRun must be a boolean value' }); + } + + // Create new service instance and add job to queue + const ttlCleanupService = new TTLCleanupService(); + const job = await ttlCleanupService.ttlCleanupQueue.add('manual-ttl-cleanup', { dryRun }); + + logger.info(`[API] TTL cleanup job triggered manually (job ID: ${job.id}, dryRun: ${dryRun})`); + + return res.status(200).json({ + message: 'TTL cleanup job triggered successfully', + jobId: job.id, + dryRun, + }); + } catch (error) { + logger.error(`[API] Error occurred triggering TTL cleanup: \n ${error}`); + return res.status(500).json({ error: 'Unable to trigger TTL cleanup job' }); + } +} From c33355544d8342135f5dd19be8de0dc4a3f85606 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 5 Jan 2026 14:33:39 -0800 Subject: [PATCH 8/8] fix tests --- src/server/services/__tests__/globalConfig.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/server/services/__tests__/globalConfig.test.ts b/src/server/services/__tests__/globalConfig.test.ts index 4cba1e3f..4be891e1 100644 --- a/src/server/services/__tests__/globalConfig.test.ts +++ b/src/server/services/__tests__/globalConfig.test.ts @@ -116,6 +116,7 @@ describe('GlobalConfigService', () => { expect(result).toEqual({ deploy: ['lifecycle-deploy!'], disabled: ['lifecycle-disabled!'], + keep: ['lifecycle-keep!'], statusComments: ['lifecycle-status-comments!'], defaultStatusComments: true, defaultControlComments: true, @@ -133,6 +134,7 @@ describe('GlobalConfigService', () => { expect(result).toEqual({ deploy: ['lifecycle-deploy!'], disabled: ['lifecycle-disabled!'], + keep: ['lifecycle-keep!'], statusComments: ['lifecycle-status-comments!'], defaultStatusComments: true, defaultControlComments: true,