From d2634a55fd1b1b5ef03a05467985404a3be408b6 Mon Sep 17 00:00:00 2001 From: Aseem Shrey Date: Mon, 16 Feb 2026 18:09:31 -0500 Subject: [PATCH 1/2] feat(config): add Zod-based startup env validation across all services Add environment variable validation at startup for backend, worker, and frontend services using Zod schemas. Misconfiguration is now caught immediately with clear, actionable error messages instead of causing silent failures or cryptic runtime errors. Shared schemas (packages/shared/src/env.ts): - databaseUrlSchema, temporalConfigSchema, minioConfigSchema - secretStoreKeySchema (exactly 32 chars), kafkaBrokersSchema - stringToBoolean helper (avoids JS coercion footgun) - formatEnvErrors for readable console output Backend (backend/src/config/env.schema.ts): - Conditional: DATABASE_URL/LOG_KAFKA_BROKERS optional when SKIP_INGEST_SERVICES=true - AUTH_PROVIDER normalized via transform+pipe+catch pattern - Clerk keys required conditionally when AUTH_PROVIDER=clerk Worker (worker/src/config/env.schema.ts): - Required: DATABASE_URL, SECRET_STORE_MASTER_KEY, LOG_KAFKA_BROKERS - MinIO and Temporal configs with sensible defaults Frontend (frontend/src/config/env.schema.ts): - VITE_API_URL defaults to localhost:3211 (test compatibility) - All VITE_* vars optional with defaults - Clerk key required conditionally Test setup provides SECRET_STORE_MASTER_KEY and SKIP_INGEST_SERVICES defaults so AppModule import doesn't fail in integration tests. Existing ad-hoc checks in database.module.ts and dev.worker.ts are preserved as defense-in-depth. Signed-off-by: Aseem Shrey --- backend/src/app.module.ts | 2 + .../src/config/__tests__/env.validate.test.ts | 104 +++++++++++++ backend/src/config/env.schema.ts | 128 ++++++++++++++++ backend/src/config/env.validate.ts | 13 ++ .../src/config/__tests__/env.schema.test.ts | 82 ++++++++++ frontend/src/config/env.schema.ts | 56 +++++++ frontend/src/config/env.ts | 30 +--- packages/shared/src/__tests__/env.test.ts | 144 ++++++++++++++++++ packages/shared/src/env.ts | 72 +++++++++ packages/shared/src/index.ts | 1 + test/setup.ts | 9 ++ .../src/config/__tests__/env.validate.test.ts | 61 ++++++++ worker/src/config/env.schema.ts | 49 ++++++ worker/src/config/env.validate.ts | 13 ++ worker/src/temporal/workers/dev.worker.ts | 2 + 15 files changed, 743 insertions(+), 23 deletions(-) create mode 100644 backend/src/config/__tests__/env.validate.test.ts create mode 100644 backend/src/config/env.schema.ts create mode 100644 backend/src/config/env.validate.ts create mode 100644 frontend/src/config/__tests__/env.schema.test.ts create mode 100644 frontend/src/config/env.schema.ts create mode 100644 packages/shared/src/__tests__/env.test.ts create mode 100644 packages/shared/src/env.ts create mode 100644 worker/src/config/__tests__/env.validate.test.ts create mode 100644 worker/src/config/env.schema.ts create mode 100644 worker/src/config/env.validate.ts diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 0c751013f..2d0553d99 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -10,6 +10,7 @@ import { AppController } from './app.controller'; import { AppService } from './app.service'; import { authConfig } from './config/auth.config'; import { opensearchConfig } from './config/opensearch.config'; +import { validateBackendEnv } from './config/env.validate'; import { OpenSearchModule } from './config/opensearch.module'; import { AgentsModule } from './agents/agents.module'; import { AuthModule } from './auth/auth.module'; @@ -76,6 +77,7 @@ function getEnvFilePaths(): string[] { isGlobal: true, envFilePath: getEnvFilePaths(), load: [authConfig, opensearchConfig], + validate: validateBackendEnv, }), ThrottlerModule.forRootAsync({ useFactory: () => { diff --git a/backend/src/config/__tests__/env.validate.test.ts b/backend/src/config/__tests__/env.validate.test.ts new file mode 100644 index 000000000..42353eddb --- /dev/null +++ b/backend/src/config/__tests__/env.validate.test.ts @@ -0,0 +1,104 @@ +import { describe, it, expect } from 'bun:test'; +import { backendEnvSchema } from '../env.schema'; + +/** Minimal valid backend env config */ +function validEnv(overrides: Record = {}): Record { + return { + DATABASE_URL: 'postgresql://user:pass@localhost:5432/db', + SECRET_STORE_MASTER_KEY: 'a'.repeat(32), + LOG_KAFKA_BROKERS: 'localhost:9092', + ...overrides, + }; +} + +describe('backendEnvSchema', () => { + it('accepts a valid full config', () => { + const result = backendEnvSchema.safeParse(validEnv()); + expect(result.success).toBe(true); + }); + + it('fails when DATABASE_URL is missing (normal mode)', () => { + const { DATABASE_URL, ...rest } = validEnv(); + const result = backendEnvSchema.safeParse(rest); + expect(result.success).toBe(false); + if (!result.success) { + const paths = result.error.issues.map((i) => i.path.join('.')); + expect(paths).toContain('DATABASE_URL'); + } + }); + + it('passes when SKIP_INGEST_SERVICES=true without DATABASE_URL and LOG_KAFKA_BROKERS', () => { + const result = backendEnvSchema.safeParse({ + SECRET_STORE_MASTER_KEY: 'a'.repeat(32), + SKIP_INGEST_SERVICES: 'true', + }); + expect(result.success).toBe(true); + }); + + it('fails when AUTH_PROVIDER=clerk without CLERK keys', () => { + const result = backendEnvSchema.safeParse(validEnv({ AUTH_PROVIDER: 'clerk' })); + expect(result.success).toBe(false); + if (!result.success) { + const paths = result.error.issues.map((i) => i.path.join('.')); + expect(paths).toContain('CLERK_SECRET_KEY'); + expect(paths).toContain('CLERK_PUBLISHABLE_KEY'); + } + }); + + it('passes when AUTH_PROVIDER=clerk with both CLERK keys', () => { + const result = backendEnvSchema.safeParse( + validEnv({ + AUTH_PROVIDER: 'clerk', + CLERK_SECRET_KEY: 'sk_test_xxx', + CLERK_PUBLISHABLE_KEY: 'pk_test_xxx', + }), + ); + expect(result.success).toBe(true); + }); + + it('normalizes " LOCAL " to "local"', () => { + const result = backendEnvSchema.safeParse(validEnv({ AUTH_PROVIDER: ' LOCAL ' })); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.AUTH_PROVIDER).toBe('local'); + } + }); + + it('falls back unknown AUTH_PROVIDER to "local"', () => { + const result = backendEnvSchema.safeParse(validEnv({ AUTH_PROVIDER: 'weird' })); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.AUTH_PROVIDER).toBe('local'); + } + }); + + it('normalizes "Clerk" to "clerk"', () => { + const result = backendEnvSchema.safeParse( + validEnv({ + AUTH_PROVIDER: 'Clerk', + CLERK_SECRET_KEY: 'sk_test_xxx', + CLERK_PUBLISHABLE_KEY: 'pk_test_xxx', + }), + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.AUTH_PROVIDER).toBe('clerk'); + } + }); + + it('coerces PORT string to number', () => { + const result = backendEnvSchema.safeParse(validEnv({ PORT: '3211' })); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.PORT).toBe(3211); + } + }); + + it('defaults PORT to 3211', () => { + const result = backendEnvSchema.safeParse(validEnv()); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.PORT).toBe(3211); + } + }); +}); diff --git a/backend/src/config/env.schema.ts b/backend/src/config/env.schema.ts new file mode 100644 index 000000000..0981d262c --- /dev/null +++ b/backend/src/config/env.schema.ts @@ -0,0 +1,128 @@ +import { z } from 'zod'; +import { + databaseUrlSchema, + temporalConfigSchema, + secretStoreKeySchema, + stringToBoolean, +} from '@shipsec/shared'; + +/** + * AUTH_PROVIDER: trims, lowercases, and defaults unknown values to 'local'. + * Preserves the tolerant normalization from auth.config.ts. + */ +const authProviderSchema = z + .string() + .optional() + .default('local') + .transform((v) => v.trim().toLowerCase()) + .pipe(z.enum(['local', 'clerk']).catch('local')); + +export const backendEnvSchema = z + .object({ + // --- Conditionally required (depend on SKIP_INGEST_SERVICES) --- + DATABASE_URL: z.string().optional(), + LOG_KAFKA_BROKERS: z.string().optional(), + + // --- Required --- + SECRET_STORE_MASTER_KEY: secretStoreKeySchema, + + // --- With defaults --- + PORT: z.coerce.number().optional().default(3211), + HOST: z.string().optional().default('0.0.0.0'), + SKIP_INGEST_SERVICES: stringToBoolean(false), + ENABLE_INGEST_SERVICES: stringToBoolean(true), + + // --- Auth --- + AUTH_PROVIDER: authProviderSchema, + CLERK_SECRET_KEY: z.string().optional(), + CLERK_PUBLISHABLE_KEY: z.string().optional(), + AUTH_LOCAL_ALLOW_UNAUTHENTICATED: stringToBoolean(true), + AUTH_LOCAL_API_KEY: z.string().optional().default(''), + ADMIN_USERNAME: z.string().optional().default('admin'), + ADMIN_PASSWORD: z.string().optional().default('admin'), + + // --- Optional services --- + REDIS_URL: z.string().optional(), + SESSION_SECRET: z.string().optional().default(''), + WEBHOOK_BASE_URL: z.string().optional(), + + // --- OpenSearch --- + OPENSEARCH_URL: z.string().optional(), + OPENSEARCH_USERNAME: z.string().optional(), + OPENSEARCH_PASSWORD: z.string().optional(), + OPENSEARCH_DASHBOARDS_URL: z.string().optional().default(''), + + // --- Loki --- + LOKI_URL: z.string().optional(), + LOKI_TENANT_ID: z.string().optional().default(''), + LOKI_USERNAME: z.string().optional().default(''), + LOKI_PASSWORD: z.string().optional().default(''), + + // --- MinIO --- + MINIO_ROOT_USER: z.string().optional(), + MINIO_ROOT_PASSWORD: z.string().optional(), + + // --- GitHub OAuth --- + GITHUB_OAUTH_CLIENT_ID: z.string().optional(), + GITHUB_OAUTH_CLIENT_SECRET: z.string().optional(), + + // --- Zoom OAuth --- + ZOOM_OAUTH_CLIENT_ID: z.string().optional(), + ZOOM_OAUTH_CLIENT_SECRET: z.string().optional(), + + // --- Platform --- + PLATFORM_API_URL: z.string().optional().default(''), + PLATFORM_SERVICE_TOKEN: z.string().optional().default(''), + PLATFORM_API_TIMEOUT_MS: z.string().optional().default(''), + + // --- Temporal --- + TEMPORAL_BOOTSTRAP_DEMO: stringToBoolean(false), + }) + .merge(temporalConfigSchema) + .superRefine((data, ctx) => { + // DATABASE_URL is required unless SKIP_INGEST_SERVICES is true + if (!data.SKIP_INGEST_SERVICES) { + if (!data.DATABASE_URL) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['DATABASE_URL'], + message: 'DATABASE_URL is required (set SKIP_INGEST_SERVICES=true to skip)', + }); + } else { + const parsed = databaseUrlSchema.safeParse(data.DATABASE_URL); + if (!parsed.success) { + for (const issue of parsed.error.issues) { + ctx.addIssue({ ...issue, path: ['DATABASE_URL'] }); + } + } + } + + if (!data.LOG_KAFKA_BROKERS) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['LOG_KAFKA_BROKERS'], + message: 'LOG_KAFKA_BROKERS is required (set SKIP_INGEST_SERVICES=true to skip)', + }); + } + } + + // Clerk keys required when AUTH_PROVIDER is clerk + if (data.AUTH_PROVIDER === 'clerk') { + if (!data.CLERK_SECRET_KEY) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['CLERK_SECRET_KEY'], + message: 'CLERK_SECRET_KEY is required when AUTH_PROVIDER=clerk', + }); + } + if (!data.CLERK_PUBLISHABLE_KEY) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['CLERK_PUBLISHABLE_KEY'], + message: 'CLERK_PUBLISHABLE_KEY is required when AUTH_PROVIDER=clerk', + }); + } + } + }); + +export type BackendEnvConfig = z.infer; diff --git a/backend/src/config/env.validate.ts b/backend/src/config/env.validate.ts new file mode 100644 index 000000000..6cf912abe --- /dev/null +++ b/backend/src/config/env.validate.ts @@ -0,0 +1,13 @@ +import { formatEnvErrors } from '@shipsec/shared'; +import { backendEnvSchema, type BackendEnvConfig } from './env.schema'; + +export function validateBackendEnv(config: Record): BackendEnvConfig { + const result = backendEnvSchema.safeParse(config); + if (!result.success) { + console.error('\n❌ Backend environment validation failed:\n'); + console.error(formatEnvErrors(result.error)); + console.error('\nSee backend/.env.example for reference.\n'); + throw new Error('Invalid backend environment configuration'); + } + return result.data; +} diff --git a/frontend/src/config/__tests__/env.schema.test.ts b/frontend/src/config/__tests__/env.schema.test.ts new file mode 100644 index 000000000..a8b95428e --- /dev/null +++ b/frontend/src/config/__tests__/env.schema.test.ts @@ -0,0 +1,82 @@ +import { describe, it, expect } from 'bun:test'; +import { frontendEnvSchema } from '../env.schema'; + +describe('frontendEnvSchema', () => { + it('accepts a valid config', () => { + const result = frontendEnvSchema.safeParse({ + VITE_API_URL: 'http://localhost:3211', + VITE_ENABLE_CONNECTIONS: 'true', + }); + expect(result.success).toBe(true); + }); + + it('empty object passes and VITE_API_URL defaults to http://localhost:3211', () => { + const result = frontendEnvSchema.safeParse({}); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.VITE_API_URL).toBe('http://localhost:3211'); + } + }); + + it('VITE_API_URL=undefined defaults to http://localhost:3211', () => { + const result = frontendEnvSchema.safeParse({ VITE_API_URL: undefined }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.VITE_API_URL).toBe('http://localhost:3211'); + } + }); + + it('VITE_ENABLE_CONNECTIONS="false" → false (not truthy)', () => { + const result = frontendEnvSchema.safeParse({ VITE_ENABLE_CONNECTIONS: 'false' }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.VITE_ENABLE_CONNECTIONS).toBe(false); + } + }); + + it('VITE_ENABLE_CONNECTIONS=undefined → false', () => { + const result = frontendEnvSchema.safeParse({}); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.VITE_ENABLE_CONNECTIONS).toBe(false); + } + }); + + it('VITE_DISABLE_ANALYTICS="true" → true', () => { + const result = frontendEnvSchema.safeParse({ VITE_DISABLE_ANALYTICS: 'true' }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.VITE_DISABLE_ANALYTICS).toBe(true); + } + }); + + it('all optional vars get defaults', () => { + const result = frontendEnvSchema.safeParse({}); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.VITE_FRONTEND_BRANCH).toBe(''); + expect(result.data.VITE_BACKEND_BRANCH).toBe(''); + expect(result.data.VITE_GIT_SHA).toBe(''); + expect(result.data.VITE_LOGO_DEV_PUBLIC_KEY).toBe(''); + expect(result.data.VITE_OPENSEARCH_DASHBOARDS_URL).toBe(''); + expect(result.data.VITE_ENABLE_IT_OPS).toBe(false); + } + }); + + it('fails when VITE_AUTH_PROVIDER=clerk without VITE_CLERK_PUBLISHABLE_KEY', () => { + const result = frontendEnvSchema.safeParse({ VITE_AUTH_PROVIDER: 'clerk' }); + expect(result.success).toBe(false); + if (!result.success) { + const paths = result.error.issues.map((i) => i.path.join('.')); + expect(paths).toContain('VITE_CLERK_PUBLISHABLE_KEY'); + } + }); + + it('passes when VITE_AUTH_PROVIDER=clerk with VITE_CLERK_PUBLISHABLE_KEY', () => { + const result = frontendEnvSchema.safeParse({ + VITE_AUTH_PROVIDER: 'clerk', + VITE_CLERK_PUBLISHABLE_KEY: 'pk_test_xxx', + }); + expect(result.success).toBe(true); + }); +}); diff --git a/frontend/src/config/env.schema.ts b/frontend/src/config/env.schema.ts new file mode 100644 index 000000000..a96a2c3e6 --- /dev/null +++ b/frontend/src/config/env.schema.ts @@ -0,0 +1,56 @@ +import { z } from 'zod'; + +/** + * Explicit string→boolean for VITE_* flags. + * Accepts 'true', 'false', '', or undefined. Never uses z.coerce.boolean(). + */ +function viteBoolean(defaultValue = false) { + return z + .enum(['true', 'false', '']) + .optional() + .default(defaultValue ? 'true' : 'false') + .transform((v) => v === 'true'); +} + +export const frontendEnvSchema = z + .object({ + // API URL — defaults to localhost for dev/test compatibility + VITE_API_URL: z.string().optional().default('http://localhost:3211'), + + // Application metadata + VITE_APP_NAME: z.string().optional().default(''), + VITE_APP_VERSION: z.string().optional().default(''), + VITE_FRONTEND_BRANCH: z.string().optional().default(''), + VITE_BACKEND_BRANCH: z.string().optional().default(''), + VITE_GIT_SHA: z.string().optional().default(''), + + // Feature flags + VITE_ENABLE_CONNECTIONS: viteBoolean(false), + VITE_ENABLE_IT_OPS: viteBoolean(false), + VITE_DISABLE_ANALYTICS: viteBoolean(false), + + // Third-party integrations + VITE_LOGO_DEV_PUBLIC_KEY: z.string().optional().default(''), + VITE_PUBLIC_POSTHOG_KEY: z.string().optional().default(''), + VITE_PUBLIC_POSTHOG_HOST: z.string().optional().default(''), + VITE_OPENSEARCH_DASHBOARDS_URL: z.string().optional().default(''), + + // Auth + VITE_AUTH_PROVIDER: z.string().optional().default(''), + VITE_CLERK_PUBLISHABLE_KEY: z.string().optional().default(''), + VITE_CLERK_JWT_TEMPLATE: z.string().optional().default(''), + VITE_API_AUTH_PROVIDER: z.string().optional().default(''), + }) + .superRefine((data, ctx) => { + // If auth provider is clerk, VITE_CLERK_PUBLISHABLE_KEY is required + const provider = data.VITE_AUTH_PROVIDER.trim().toLowerCase(); + if (provider === 'clerk' && !data.VITE_CLERK_PUBLISHABLE_KEY) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['VITE_CLERK_PUBLISHABLE_KEY'], + message: 'VITE_CLERK_PUBLISHABLE_KEY is required when VITE_AUTH_PROVIDER=clerk', + }); + } + }); + +export type FrontendEnvConfig = z.infer; diff --git a/frontend/src/config/env.ts b/frontend/src/config/env.ts index a9eb9fa57..31eedf15c 100644 --- a/frontend/src/config/env.ts +++ b/frontend/src/config/env.ts @@ -1,25 +1,9 @@ -// Centralized access to selected Vite env vars used in the UI. -// Keep this minimal and typed; provide empty-string fallbacks so UI never breaks. +import { frontendEnvSchema } from './env.schema'; -interface FrontendEnv { - VITE_FRONTEND_BRANCH: string; - VITE_BACKEND_BRANCH: string; - VITE_GIT_SHA: string; - VITE_LOGO_DEV_PUBLIC_KEY: string; - VITE_ENABLE_CONNECTIONS: boolean; - VITE_ENABLE_IT_OPS: boolean; - VITE_API_URL: string; - VITE_OPENSEARCH_DASHBOARDS_URL: string; +const result = frontendEnvSchema.safeParse(import.meta.env); +if (!result.success) { + const msg = '❌ Frontend env validation failed'; + console.error(msg, result.error.issues); + throw new Error(msg); } - -export const env: FrontendEnv = { - VITE_FRONTEND_BRANCH: (import.meta.env.VITE_FRONTEND_BRANCH as string | undefined) ?? '', - VITE_BACKEND_BRANCH: (import.meta.env.VITE_BACKEND_BRANCH as string | undefined) ?? '', - VITE_GIT_SHA: (import.meta.env.VITE_GIT_SHA as string | undefined) ?? '', - VITE_LOGO_DEV_PUBLIC_KEY: (import.meta.env.VITE_LOGO_DEV_PUBLIC_KEY as string | undefined) ?? '', - VITE_ENABLE_CONNECTIONS: import.meta.env.VITE_ENABLE_CONNECTIONS === 'true', - VITE_ENABLE_IT_OPS: import.meta.env.VITE_ENABLE_IT_OPS === 'true', - VITE_API_URL: (import.meta.env.VITE_API_URL as string | undefined) ?? '', - VITE_OPENSEARCH_DASHBOARDS_URL: - (import.meta.env.VITE_OPENSEARCH_DASHBOARDS_URL as string | undefined) ?? '', -}; +export const env = result.data; diff --git a/packages/shared/src/__tests__/env.test.ts b/packages/shared/src/__tests__/env.test.ts new file mode 100644 index 000000000..51c7a022d --- /dev/null +++ b/packages/shared/src/__tests__/env.test.ts @@ -0,0 +1,144 @@ +import { describe, it, expect } from 'bun:test'; +import { + databaseUrlSchema, + secretStoreKeySchema, + kafkaBrokersSchema, + minioConfigSchema, + stringToBoolean, + formatEnvErrors, +} from '../env.js'; +import { z } from 'zod'; + +describe('databaseUrlSchema', () => { + it('accepts a valid postgresql URL', () => { + const result = databaseUrlSchema.safeParse('postgresql://user:pass@localhost:5432/db'); + expect(result.success).toBe(true); + }); + + it('rejects a non-postgresql URL', () => { + const result = databaseUrlSchema.safeParse('mysql://user:pass@localhost/db'); + expect(result.success).toBe(false); + }); + + it('rejects an empty string', () => { + const result = databaseUrlSchema.safeParse(''); + expect(result.success).toBe(false); + }); +}); + +describe('secretStoreKeySchema', () => { + it('accepts exactly 32 characters', () => { + const result = secretStoreKeySchema.safeParse('a'.repeat(32)); + expect(result.success).toBe(true); + }); + + it('rejects 31 characters', () => { + const result = secretStoreKeySchema.safeParse('a'.repeat(31)); + expect(result.success).toBe(false); + }); + + it('rejects 33 characters', () => { + const result = secretStoreKeySchema.safeParse('a'.repeat(33)); + expect(result.success).toBe(false); + }); +}); + +describe('kafkaBrokersSchema', () => { + it('parses comma-separated brokers', () => { + const result = kafkaBrokersSchema.safeParse('a,b,c'); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data).toEqual(['a', 'b', 'c']); + } + }); + + it('trims whitespace around brokers', () => { + const result = kafkaBrokersSchema.safeParse(' host1:9092 , host2:9092 '); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data).toEqual(['host1:9092', 'host2:9092']); + } + }); + + it('rejects an empty string', () => { + const result = kafkaBrokersSchema.safeParse(''); + expect(result.success).toBe(false); + }); +}); + +describe('minioConfigSchema', () => { + it('coerces MINIO_PORT string to number', () => { + const result = minioConfigSchema.safeParse({ MINIO_PORT: '9000' }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.MINIO_PORT).toBe(9000); + } + }); + + it('defaults MINIO_ENDPOINT to localhost', () => { + const result = minioConfigSchema.safeParse({}); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.MINIO_ENDPOINT).toBe('localhost'); + } + }); + + it('parses MINIO_USE_SSL=true as boolean true', () => { + const result = minioConfigSchema.safeParse({ MINIO_USE_SSL: 'true' }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.MINIO_USE_SSL).toBe(true); + } + }); + + it('parses MINIO_USE_SSL=false as boolean false', () => { + const result = minioConfigSchema.safeParse({ MINIO_USE_SSL: 'false' }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.MINIO_USE_SSL).toBe(false); + } + }); +}); + +describe('stringToBoolean', () => { + const schema = stringToBoolean(); + + it('"true" → true', () => { + expect(schema.parse('true')).toBe(true); + }); + + it('"false" → false', () => { + expect(schema.parse('false')).toBe(false); + }); + + it('empty string → false (default)', () => { + expect(schema.parse('')).toBe(false); + }); + + it('undefined → false (default)', () => { + expect(schema.parse(undefined)).toBe(false); + }); + + it('respects custom default of true', () => { + const trueDefault = stringToBoolean(true); + expect(trueDefault.parse(undefined)).toBe(true); + }); +}); + +describe('formatEnvErrors', () => { + it('produces human-readable output', () => { + const testSchema = z.object({ + FOO: z.string({ error: 'FOO is required' }), + BAR: z.number({ error: 'BAR must be a number' }), + }); + const result = testSchema.safeParse({}); + expect(result.success).toBe(false); + if (!result.success) { + const output = formatEnvErrors(result.error); + expect(output).toContain('Variable'); + expect(output).toContain('Error'); + expect(output).toContain('FOO'); + expect(output).toContain('BAR'); + } + }); +}); diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts new file mode 100644 index 000000000..42c9d7cc7 --- /dev/null +++ b/packages/shared/src/env.ts @@ -0,0 +1,72 @@ +import { z } from 'zod'; + +// --------------------------------------------------------------------------- +// Shared helpers +// --------------------------------------------------------------------------- + +/** + * Explicit string→boolean coercion that avoids the JS footgun where + * `Boolean("false")` is `true`. Accepts 'true', 'false', '', or undefined. + */ +export function stringToBoolean(defaultValue: boolean = false) { + return z + .enum(['true', 'false', '']) + .optional() + .default(defaultValue ? 'true' : 'false') + .transform((v) => v === 'true'); +} + +/** + * Format Zod errors into a readable table for console output. + */ +export function formatEnvErrors(error: z.ZodError): string { + const lines: string[] = []; + const maxVarLen = Math.max(...error.issues.map((i) => i.path.join('.').length || 3), 8); + + lines.push(`${'Variable'.padEnd(maxVarLen)} Error`); + lines.push(`${'─'.repeat(maxVarLen)} ${'─'.repeat(40)}`); + + for (const issue of error.issues) { + const varName = issue.path.join('.') || '(root)'; + lines.push(`${varName.padEnd(maxVarLen)} ${issue.message}`); + } + + return lines.join('\n'); +} + +// --------------------------------------------------------------------------- +// Reusable schemas for env vars shared across services +// --------------------------------------------------------------------------- + +/** DATABASE_URL — must start with postgresql:// */ +export const databaseUrlSchema = z + .string({ error: 'DATABASE_URL is required' }) + .startsWith('postgresql://', { message: 'DATABASE_URL must start with postgresql://' }); + +/** Temporal connection configuration */ +export const temporalConfigSchema = z.object({ + TEMPORAL_ADDRESS: z.string().optional().default('localhost:7233'), + TEMPORAL_NAMESPACE: z.string().optional().default('shipsec-dev'), + TEMPORAL_TASK_QUEUE: z.string().optional().default('shipsec-dev'), +}); + +/** MinIO / S3-compatible storage configuration */ +export const minioConfigSchema = z.object({ + MINIO_ENDPOINT: z.string().optional().default('localhost'), + MINIO_PORT: z.coerce.number().optional().default(9000), + MINIO_ACCESS_KEY: z.string().optional().default('minioadmin'), + MINIO_SECRET_KEY: z.string().optional().default('minioadmin'), + MINIO_USE_SSL: stringToBoolean(false), + MINIO_BUCKET_NAME: z.string().optional().default('shipsec-files'), +}); + +/** SECRET_STORE_MASTER_KEY — exactly 32 characters */ +export const secretStoreKeySchema = z + .string({ error: 'SECRET_STORE_MASTER_KEY is required' }) + .length(32, { message: 'SECRET_STORE_MASTER_KEY must be exactly 32 characters' }); + +/** LOG_KAFKA_BROKERS — comma-separated string → array of strings */ +export const kafkaBrokersSchema = z + .string({ error: 'LOG_KAFKA_BROKERS is required' }) + .min(1, 'LOG_KAFKA_BROKERS must not be empty') + .transform((v) => v.split(',').map((s) => s.trim()).filter(Boolean)); diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index cee4a930b..01b443b0d 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -6,3 +6,4 @@ export * from './destinations.js'; export * from './schedules.js'; export * from './webhooks.js'; export * from './mcp.js'; +export * from './env.js'; diff --git a/test/setup.ts b/test/setup.ts index b6d98ef3e..15b7f9a4b 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -1 +1,10 @@ +// Provide test defaults so the backend env validation doesn't throw when +// AppModule is imported by integration/e2e tests that skip actual execution. +if (!process.env.SECRET_STORE_MASTER_KEY) { + process.env.SECRET_STORE_MASTER_KEY = 'aaaaaaaaaabbbbbbbbbbccccccccccdd'; +} +if (!process.env.SKIP_INGEST_SERVICES) { + process.env.SKIP_INGEST_SERVICES = 'true'; +} + import '../frontend/src/test/setup.ts' diff --git a/worker/src/config/__tests__/env.validate.test.ts b/worker/src/config/__tests__/env.validate.test.ts new file mode 100644 index 000000000..880af24c9 --- /dev/null +++ b/worker/src/config/__tests__/env.validate.test.ts @@ -0,0 +1,61 @@ +import { describe, it, expect } from 'bun:test'; +import { workerEnvSchema } from '../env.schema'; + +/** Minimal valid worker env config */ +function validEnv(overrides: Record = {}): Record { + return { + DATABASE_URL: 'postgresql://user:pass@localhost:5432/db', + SECRET_STORE_MASTER_KEY: 'a'.repeat(32), + LOG_KAFKA_BROKERS: 'localhost:9092', + ...overrides, + }; +} + +describe('workerEnvSchema', () => { + it('accepts a valid config', () => { + const result = workerEnvSchema.safeParse(validEnv()); + expect(result.success).toBe(true); + }); + + it('fails when DATABASE_URL is missing', () => { + const { DATABASE_URL, ...rest } = validEnv(); + const result = workerEnvSchema.safeParse(rest); + expect(result.success).toBe(false); + }); + + it('fails when SECRET_STORE_MASTER_KEY is missing', () => { + const { SECRET_STORE_MASTER_KEY, ...rest } = validEnv(); + const result = workerEnvSchema.safeParse(rest); + expect(result.success).toBe(false); + }); + + it('fails when LOG_KAFKA_BROKERS is missing', () => { + const { LOG_KAFKA_BROKERS, ...rest } = validEnv(); + const result = workerEnvSchema.safeParse(rest); + expect(result.success).toBe(false); + }); + + it('defaults MINIO_ENDPOINT to localhost', () => { + const result = workerEnvSchema.safeParse(validEnv()); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.MINIO_ENDPOINT).toBe('localhost'); + } + }); + + it('defaults BACKEND_URL to http://localhost:3211', () => { + const result = workerEnvSchema.safeParse(validEnv()); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.BACKEND_URL).toBe('http://localhost:3211'); + } + }); + + it('parses LOG_KAFKA_BROKERS into array', () => { + const result = workerEnvSchema.safeParse(validEnv({ LOG_KAFKA_BROKERS: 'a:9092,b:9092' })); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.LOG_KAFKA_BROKERS).toEqual(['a:9092', 'b:9092']); + } + }); +}); diff --git a/worker/src/config/env.schema.ts b/worker/src/config/env.schema.ts new file mode 100644 index 000000000..c662744ce --- /dev/null +++ b/worker/src/config/env.schema.ts @@ -0,0 +1,49 @@ +import { z } from 'zod'; +import { + databaseUrlSchema, + temporalConfigSchema, + minioConfigSchema, + secretStoreKeySchema, + kafkaBrokersSchema, +} from '@shipsec/shared'; + +export const workerEnvSchema = z + .object({ + // --- Required --- + DATABASE_URL: databaseUrlSchema, + SECRET_STORE_MASTER_KEY: secretStoreKeySchema, + LOG_KAFKA_BROKERS: kafkaBrokersSchema, + + // --- With defaults --- + BACKEND_URL: z.string().optional().default('http://localhost:3211'), + + // --- Optional Kafka client IDs --- + EVENT_KAFKA_CLIENT_ID: z.string().optional().default('shipsec-worker-events'), + AGENT_TRACE_KAFKA_CLIENT_ID: z.string().optional().default('shipsec-worker-agent-trace'), + NODE_IO_KAFKA_CLIENT_ID: z.string().optional().default('shipsec-worker-node-io'), + LOG_KAFKA_CLIENT_ID: z.string().optional().default('shipsec-worker'), + + // --- Terminal Redis --- + TERMINAL_REDIS_URL: z.string().optional(), + TERMINAL_REDIS_MAXLEN: z.coerce.number().optional().default(5000), + + // --- Loki --- + LOKI_URL: z.string().optional(), + LOKI_TENANT_ID: z.string().optional().default(''), + LOKI_USERNAME: z.string().optional().default(''), + LOKI_PASSWORD: z.string().optional().default(''), + + // --- OpenSearch --- + OPENSEARCH_URL: z.string().optional(), + OPENSEARCH_USERNAME: z.string().optional(), + OPENSEARCH_PASSWORD: z.string().optional(), + OPENSEARCH_DASHBOARDS_URL: z.string().optional().default(''), + + // --- AI provider keys (all optional) --- + OPENAI_API_KEY: z.string().optional(), + ANTHROPIC_API_KEY: z.string().optional(), + }) + .merge(temporalConfigSchema) + .merge(minioConfigSchema); + +export type WorkerEnvConfig = z.infer; diff --git a/worker/src/config/env.validate.ts b/worker/src/config/env.validate.ts new file mode 100644 index 000000000..4b960ad30 --- /dev/null +++ b/worker/src/config/env.validate.ts @@ -0,0 +1,13 @@ +import { formatEnvErrors } from '@shipsec/shared'; +import { workerEnvSchema, type WorkerEnvConfig } from './env.schema'; + +export function validateWorkerEnv(env: Record): WorkerEnvConfig { + const result = workerEnvSchema.safeParse(env); + if (!result.success) { + console.error('\n❌ Worker environment validation failed:\n'); + console.error(formatEnvErrors(result.error)); + console.error('\nSee worker/.env.example for reference.\n'); + throw new Error('Invalid worker environment configuration'); + } + return result.data; +} diff --git a/worker/src/temporal/workers/dev.worker.ts b/worker/src/temporal/workers/dev.worker.ts index d6383c8bc..6e2495cc6 100644 --- a/worker/src/temporal/workers/dev.worker.ts +++ b/worker/src/temporal/workers/dev.worker.ts @@ -56,6 +56,7 @@ import { ConfigurationError } from '@shipsec/component-sdk'; import { getTopicResolver } from '../../common/kafka-topic-resolver'; import * as schema from '../../adapters/schema'; import { logHeartbeat } from '../../utils/debug-logger'; +import { validateWorkerEnv } from '../../config/env.validate'; // Load environment variables from instance-specific env if set, otherwise fall back // to the worker's default `.env`. @@ -66,6 +67,7 @@ const instanceEnvPath = instanceNum : undefined; config({ path: instanceEnvPath ?? join(workerRoot, '.env') }); +validateWorkerEnv(process.env); if (typeof globalThis.crypto === 'undefined') { Object.defineProperty(globalThis, 'crypto', { From cb595fc6b077408f4994333cf6b61764ed771dfb Mon Sep 17 00:00:00 2001 From: Aseem Shrey Date: Mon, 16 Feb 2026 18:25:01 -0500 Subject: [PATCH 2/2] fix(config): gate ingest validation on ENABLE_INGEST_SERVICES and stop forcing SKIP_INGEST_SERVICES in test setup Address PR review feedback: - superRefine now checks both ENABLE_INGEST_SERVICES and SKIP_INGEST_SERVICES, matching the runtime guard in node-io.module.ts / trace.module.ts - test/setup.ts provides dummy DATABASE_URL and LOG_KAFKA_BROKERS instead of setting SKIP_INGEST_SERVICES=true, so integration tests exercise real code paths Signed-off-by: Aseem Shrey --- backend/src/config/__tests__/env.validate.test.ts | 8 ++++++++ backend/src/config/env.schema.ts | 14 +++++++++----- test/setup.ts | 10 ++++++++-- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/backend/src/config/__tests__/env.validate.test.ts b/backend/src/config/__tests__/env.validate.test.ts index 42353eddb..15e4e3f20 100644 --- a/backend/src/config/__tests__/env.validate.test.ts +++ b/backend/src/config/__tests__/env.validate.test.ts @@ -35,6 +35,14 @@ describe('backendEnvSchema', () => { expect(result.success).toBe(true); }); + it('passes when ENABLE_INGEST_SERVICES=false without DATABASE_URL and LOG_KAFKA_BROKERS', () => { + const result = backendEnvSchema.safeParse({ + SECRET_STORE_MASTER_KEY: 'a'.repeat(32), + ENABLE_INGEST_SERVICES: 'false', + }); + expect(result.success).toBe(true); + }); + it('fails when AUTH_PROVIDER=clerk without CLERK keys', () => { const result = backendEnvSchema.safeParse(validEnv({ AUTH_PROVIDER: 'clerk' })); expect(result.success).toBe(false); diff --git a/backend/src/config/env.schema.ts b/backend/src/config/env.schema.ts index 0981d262c..df12cf9db 100644 --- a/backend/src/config/env.schema.ts +++ b/backend/src/config/env.schema.ts @@ -19,7 +19,7 @@ const authProviderSchema = z export const backendEnvSchema = z .object({ - // --- Conditionally required (depend on SKIP_INGEST_SERVICES) --- + // --- Conditionally required (depend on ingest-services flags) --- DATABASE_URL: z.string().optional(), LOG_KAFKA_BROKERS: z.string().optional(), @@ -80,13 +80,16 @@ export const backendEnvSchema = z }) .merge(temporalConfigSchema) .superRefine((data, ctx) => { - // DATABASE_URL is required unless SKIP_INGEST_SERVICES is true - if (!data.SKIP_INGEST_SERVICES) { + // Match the runtime guard in node-io.module.ts / trace.module.ts: + // ingestEnabled = ENABLE_INGEST_SERVICES !== false && SKIP_INGEST_SERVICES !== true + const ingestRequired = data.ENABLE_INGEST_SERVICES && !data.SKIP_INGEST_SERVICES; + if (ingestRequired) { if (!data.DATABASE_URL) { ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['DATABASE_URL'], - message: 'DATABASE_URL is required (set SKIP_INGEST_SERVICES=true to skip)', + message: + 'DATABASE_URL is required (set SKIP_INGEST_SERVICES=true or ENABLE_INGEST_SERVICES=false to skip)', }); } else { const parsed = databaseUrlSchema.safeParse(data.DATABASE_URL); @@ -101,7 +104,8 @@ export const backendEnvSchema = z ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['LOG_KAFKA_BROKERS'], - message: 'LOG_KAFKA_BROKERS is required (set SKIP_INGEST_SERVICES=true to skip)', + message: + 'LOG_KAFKA_BROKERS is required (set SKIP_INGEST_SERVICES=true or ENABLE_INGEST_SERVICES=false to skip)', }); } } diff --git a/test/setup.ts b/test/setup.ts index 15b7f9a4b..3ca90aa4b 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -1,10 +1,16 @@ // Provide test defaults so the backend env validation doesn't throw when // AppModule is imported by integration/e2e tests that skip actual execution. +// NOTE: We intentionally set dummy values instead of SKIP_INGEST_SERVICES=true +// so that DatabaseModule / ingest modules keep their real behaviour and +// integration tests that import AppModule exercise real code paths. if (!process.env.SECRET_STORE_MASTER_KEY) { process.env.SECRET_STORE_MASTER_KEY = 'aaaaaaaaaabbbbbbbbbbccccccccccdd'; } -if (!process.env.SKIP_INGEST_SERVICES) { - process.env.SKIP_INGEST_SERVICES = 'true'; +if (!process.env.DATABASE_URL) { + process.env.DATABASE_URL = 'postgresql://test:test@localhost:5432/test'; +} +if (!process.env.LOG_KAFKA_BROKERS) { + process.env.LOG_KAFKA_BROKERS = 'localhost:9092'; } import '../frontend/src/test/setup.ts'