feat: env file sync with infra config (#4545)

This commit is contained in:
Mir Arif Hasan
2024-11-26 20:22:51 +06:00
committed by GitHub
parent 80d7dd046d
commit 5dccce39b4
8 changed files with 210 additions and 59 deletions

View File

@@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "InfraConfig" DROP COLUMN "active",
ADD COLUMN "lastSyncedEnvFileValue" TEXT;

View File

@@ -214,13 +214,13 @@ enum TeamMemberRole {
}
model InfraConfig {
id String @id @default(cuid())
name String @unique
value String?
isEncrypted Boolean @default(false) // Use case: Let's say, Admin wants to store a Secret Key, but doesn't want to store it in plain text in `value` column
active Boolean @default(true) // Use case: Let's say, Admin wants to disable Google SSO, but doesn't want to delete the config
createdOn DateTime @default(now()) @db.Timestamp(3)
updatedOn DateTime @updatedAt @db.Timestamp(3)
id String @id @default(cuid())
name String @unique
value String?
lastSyncedEnvFileValue String?
isEncrypted Boolean @default(false) // Use case: Let's say, Admin wants to store a Secret Key, but doesn't want to store it in plain text in `value` column
createdOn DateTime @default(now()) @db.Timestamp(3)
updatedOn DateTime @updatedAt @db.Timestamp(3)
}
model PersonalAccessToken {

View File

@@ -13,8 +13,8 @@ import { MicrosoftStrategy } from './strategies/microsoft.strategy';
import { AuthProvider, authProviderCheck } from './helper';
import { ConfigModule, ConfigService } from '@nestjs/config';
import {
getConfiguredSSOProvidersFromInfraConfig,
isInfraConfigTablePopulated,
loadInfraConfiguration,
} from 'src/infra-config/helper';
import { InfraConfigModule } from 'src/infra-config/infra-config.module';
@@ -42,8 +42,8 @@ export class AuthModule {
return { module: AuthModule };
}
const env = await loadInfraConfiguration();
const allowedAuthProviders = env.INFRA.VITE_ALLOWED_AUTH_PROVIDERS;
const allowedAuthProviders =
await getConfiguredSSOProvidersFromInfraConfig();
const providers = [
...(authProviderCheck(AuthProvider.GOOGLE, allowedAuthProviders)

View File

@@ -55,6 +55,12 @@ export const ENV_NOT_FOUND_KEY_AUTH_PROVIDERS =
export const ENV_NOT_FOUND_KEY_DATA_ENCRYPTION_KEY =
'"DATA_ENCRYPTION_KEY" is not present in .env file';
/**
* Environment variable "DATA_ENCRYPTION_KEY" is changed in .env file
*/
export const ENV_INVALID_DATA_ENCRYPTION_KEY =
'"DATA_ENCRYPTION_KEY" value changed in .env file. Please undo the changes and restart the server';
/**
* Environment variable "VITE_ALLOWED_AUTH_PROVIDERS" is empty in .env file
*/

View File

@@ -2,17 +2,26 @@ import { AuthProvider } from 'src/auth/helper';
import {
AUTH_PROVIDER_NOT_CONFIGURED,
DATABASE_TABLE_NOT_EXIST,
ENV_INVALID_DATA_ENCRYPTION_KEY,
} from 'src/errors';
import { PrismaService } from 'src/prisma/prisma.service';
import { InfraConfigEnum } from 'src/types/InfraConfig';
import { decrypt, encrypt, throwErr } from 'src/utils';
import { randomBytes } from 'crypto';
import { InfraConfig } from '@prisma/client';
export enum ServiceStatus {
ENABLE = 'ENABLE',
DISABLE = 'DISABLE',
}
type DefaultInfraConfig = {
name: InfraConfigEnum;
value: string;
lastSyncedEnvFileValue: string;
isEncrypted: boolean;
};
const AuthProviderConfigurations = {
[AuthProvider.GOOGLE]: [
InfraConfigEnum.GOOGLE_CLIENT_ID,
@@ -33,17 +42,18 @@ const AuthProviderConfigurations = {
InfraConfigEnum.MICROSOFT_SCOPE,
InfraConfigEnum.MICROSOFT_TENANT,
],
[AuthProvider.EMAIL]: !!process.env.MAILER_USE_CUSTOM_CONFIGS
? [
InfraConfigEnum.MAILER_SMTP_HOST,
InfraConfigEnum.MAILER_SMTP_PORT,
InfraConfigEnum.MAILER_SMTP_SECURE,
InfraConfigEnum.MAILER_SMTP_USER,
InfraConfigEnum.MAILER_SMTP_PASSWORD,
InfraConfigEnum.MAILER_TLS_REJECT_UNAUTHORIZED,
InfraConfigEnum.MAILER_ADDRESS_FROM,
]
: [InfraConfigEnum.MAILER_SMTP_URL, InfraConfigEnum.MAILER_ADDRESS_FROM],
[AuthProvider.EMAIL]:
process.env.MAILER_USE_CUSTOM_CONFIGS === 'true'
? [
InfraConfigEnum.MAILER_SMTP_HOST,
InfraConfigEnum.MAILER_SMTP_PORT,
InfraConfigEnum.MAILER_SMTP_SECURE,
InfraConfigEnum.MAILER_SMTP_USER,
InfraConfigEnum.MAILER_SMTP_PASSWORD,
InfraConfigEnum.MAILER_TLS_REJECT_UNAUTHORIZED,
InfraConfigEnum.MAILER_ADDRESS_FROM,
]
: [InfraConfigEnum.MAILER_SMTP_URL, InfraConfigEnum.MAILER_ADDRESS_FROM],
};
/**
@@ -69,6 +79,9 @@ export async function loadInfraConfiguration() {
return { INFRA: environmentObject };
} catch (error) {
if (error.code === 'ERR_OSSL_BAD_DECRYPT')
throw new Error(ENV_INVALID_DATA_ENCRYPTION_KEY);
// Prisma throw error if 'Can't reach at database server' OR 'Table does not exist'
// Reason for not throwing error is, we want successful build during 'postinstall' and generate dist files
return { INFRA: {} };
@@ -79,150 +92,174 @@ export async function loadInfraConfiguration() {
* Read the default values from .env file and return them as an array
* @returns Array of default infra configs
*/
export async function getDefaultInfraConfigs(): Promise<
{ name: InfraConfigEnum; value: string; isEncrypted: boolean }[]
> {
export async function getDefaultInfraConfigs(): Promise<DefaultInfraConfig[]> {
const prisma = new PrismaService();
// Prepare rows for 'infra_config' table with default values (from .env) for each 'name'
const infraConfigDefaultObjs: {
name: InfraConfigEnum;
value: string;
isEncrypted: boolean;
}[] = [
const configuredSSOProviders = getConfiguredSSOProvidersFromEnvFile();
const generatedAnalyticsUserId = generateAnalyticsUserId();
const infraConfigDefaultObjs: DefaultInfraConfig[] = [
{
name: InfraConfigEnum.MAILER_SMTP_ENABLE,
value: process.env.MAILER_SMTP_ENABLE ?? 'true',
lastSyncedEnvFileValue: process.env.MAILER_SMTP_ENABLE ?? 'true',
isEncrypted: false,
},
{
name: InfraConfigEnum.MAILER_USE_CUSTOM_CONFIGS,
value: process.env.MAILER_USE_CUSTOM_CONFIGS ?? 'false',
lastSyncedEnvFileValue: process.env.MAILER_USE_CUSTOM_CONFIGS ?? 'false',
isEncrypted: false,
},
{
name: InfraConfigEnum.MAILER_SMTP_URL,
value: encrypt(process.env.MAILER_SMTP_URL),
lastSyncedEnvFileValue: encrypt(process.env.MAILER_SMTP_URL),
isEncrypted: true,
},
{
name: InfraConfigEnum.MAILER_ADDRESS_FROM,
value: process.env.MAILER_ADDRESS_FROM,
lastSyncedEnvFileValue: process.env.MAILER_ADDRESS_FROM,
isEncrypted: false,
},
{
name: InfraConfigEnum.MAILER_SMTP_HOST,
value: process.env.MAILER_SMTP_HOST,
lastSyncedEnvFileValue: process.env.MAILER_SMTP_HOST,
isEncrypted: false,
},
{
name: InfraConfigEnum.MAILER_SMTP_PORT,
value: process.env.MAILER_SMTP_PORT,
lastSyncedEnvFileValue: process.env.MAILER_SMTP_PORT,
isEncrypted: false,
},
{
name: InfraConfigEnum.MAILER_SMTP_SECURE,
value: process.env.MAILER_SMTP_SECURE,
lastSyncedEnvFileValue: process.env.MAILER_SMTP_SECURE,
isEncrypted: false,
},
{
name: InfraConfigEnum.MAILER_SMTP_USER,
value: process.env.MAILER_SMTP_USER,
lastSyncedEnvFileValue: process.env.MAILER_SMTP_USER,
isEncrypted: false,
},
{
name: InfraConfigEnum.MAILER_SMTP_PASSWORD,
value: encrypt(process.env.MAILER_SMTP_PASSWORD),
lastSyncedEnvFileValue: encrypt(process.env.MAILER_SMTP_PASSWORD),
isEncrypted: true,
},
{
name: InfraConfigEnum.MAILER_TLS_REJECT_UNAUTHORIZED,
value: process.env.MAILER_TLS_REJECT_UNAUTHORIZED,
lastSyncedEnvFileValue: process.env.MAILER_TLS_REJECT_UNAUTHORIZED,
isEncrypted: false,
},
{
name: InfraConfigEnum.GOOGLE_CLIENT_ID,
value: encrypt(process.env.GOOGLE_CLIENT_ID),
lastSyncedEnvFileValue: encrypt(process.env.GOOGLE_CLIENT_ID),
isEncrypted: true,
},
{
name: InfraConfigEnum.GOOGLE_CLIENT_SECRET,
value: encrypt(process.env.GOOGLE_CLIENT_SECRET),
lastSyncedEnvFileValue: encrypt(process.env.GOOGLE_CLIENT_SECRET),
isEncrypted: true,
},
{
name: InfraConfigEnum.GOOGLE_CALLBACK_URL,
value: process.env.GOOGLE_CALLBACK_URL,
lastSyncedEnvFileValue: process.env.GOOGLE_CALLBACK_URL,
isEncrypted: false,
},
{
name: InfraConfigEnum.GOOGLE_SCOPE,
value: process.env.GOOGLE_SCOPE,
lastSyncedEnvFileValue: process.env.GOOGLE_SCOPE,
isEncrypted: false,
},
{
name: InfraConfigEnum.GITHUB_CLIENT_ID,
value: encrypt(process.env.GITHUB_CLIENT_ID),
lastSyncedEnvFileValue: encrypt(process.env.GITHUB_CLIENT_ID),
isEncrypted: true,
},
{
name: InfraConfigEnum.GITHUB_CLIENT_SECRET,
value: encrypt(process.env.GITHUB_CLIENT_SECRET),
lastSyncedEnvFileValue: encrypt(process.env.GITHUB_CLIENT_SECRET),
isEncrypted: true,
},
{
name: InfraConfigEnum.GITHUB_CALLBACK_URL,
value: process.env.GITHUB_CALLBACK_URL,
lastSyncedEnvFileValue: process.env.GITHUB_CALLBACK_URL,
isEncrypted: false,
},
{
name: InfraConfigEnum.GITHUB_SCOPE,
value: process.env.GITHUB_SCOPE,
lastSyncedEnvFileValue: process.env.GITHUB_SCOPE,
isEncrypted: false,
},
{
name: InfraConfigEnum.MICROSOFT_CLIENT_ID,
value: encrypt(process.env.MICROSOFT_CLIENT_ID),
lastSyncedEnvFileValue: encrypt(process.env.MICROSOFT_CLIENT_ID),
isEncrypted: true,
},
{
name: InfraConfigEnum.MICROSOFT_CLIENT_SECRET,
value: encrypt(process.env.MICROSOFT_CLIENT_SECRET),
lastSyncedEnvFileValue: encrypt(process.env.MICROSOFT_CLIENT_SECRET),
isEncrypted: true,
},
{
name: InfraConfigEnum.MICROSOFT_CALLBACK_URL,
value: process.env.MICROSOFT_CALLBACK_URL,
lastSyncedEnvFileValue: process.env.MICROSOFT_CALLBACK_URL,
isEncrypted: false,
},
{
name: InfraConfigEnum.MICROSOFT_SCOPE,
value: process.env.MICROSOFT_SCOPE,
lastSyncedEnvFileValue: process.env.MICROSOFT_SCOPE,
isEncrypted: false,
},
{
name: InfraConfigEnum.MICROSOFT_TENANT,
value: process.env.MICROSOFT_TENANT,
lastSyncedEnvFileValue: process.env.MICROSOFT_TENANT,
isEncrypted: false,
},
{
name: InfraConfigEnum.VITE_ALLOWED_AUTH_PROVIDERS,
value: getConfiguredSSOProviders(),
value: configuredSSOProviders,
lastSyncedEnvFileValue: configuredSSOProviders,
isEncrypted: false,
},
{
name: InfraConfigEnum.ALLOW_ANALYTICS_COLLECTION,
value: false.toString(),
lastSyncedEnvFileValue: null,
isEncrypted: false,
},
{
name: InfraConfigEnum.ANALYTICS_USER_ID,
value: generateAnalyticsUserId(),
value: generatedAnalyticsUserId,
lastSyncedEnvFileValue: null,
isEncrypted: false,
},
{
name: InfraConfigEnum.IS_FIRST_TIME_INFRA_SETUP,
value: (await prisma.infraConfig.count()) === 0 ? 'true' : 'false',
lastSyncedEnvFileValue: null,
isEncrypted: false,
},
];
@@ -234,12 +271,11 @@ export async function getDefaultInfraConfigs(): Promise<
* Get the missing entries in the 'infra_config' table
* @returns Array of InfraConfig
*/
export async function getMissingInfraConfigEntries() {
export async function getMissingInfraConfigEntries(
infraConfigDefaultObjs: DefaultInfraConfig[],
) {
const prisma = new PrismaService();
const [dbInfraConfigs, infraConfigDefaultObjs] = await Promise.all([
prisma.infraConfig.findMany(),
getDefaultInfraConfigs(),
]);
const dbInfraConfigs = await prisma.infraConfig.findMany();
const missingEntries = infraConfigDefaultObjs.filter(
(config) =>
@@ -253,12 +289,11 @@ export async function getMissingInfraConfigEntries() {
* Get the encryption required entries in the 'infra_config' table
* @returns Array of InfraConfig
*/
export async function getEncryptionRequiredInfraConfigEntries() {
export async function getEncryptionRequiredInfraConfigEntries(
infraConfigDefaultObjs: DefaultInfraConfig[],
) {
const prisma = new PrismaService();
const [dbInfraConfigs, infraConfigDefaultObjs] = await Promise.all([
prisma.infraConfig.findMany(),
getDefaultInfraConfigs(),
]);
const dbInfraConfigs = await prisma.infraConfig.findMany();
const requiredEncryption = dbInfraConfigs.filter((dbConfig) => {
const defaultConfig = infraConfigDefaultObjs.find(
@@ -271,13 +306,57 @@ export async function getEncryptionRequiredInfraConfigEntries() {
return requiredEncryption;
}
/**
* Sync the 'infra_config' table with .env file
* @returns Array of InfraConfig
*/
export async function syncInfraConfigWithEnvFile() {
const prisma = new PrismaService();
const dbInfraConfigs = await prisma.infraConfig.findMany();
const updateRequiredObjs: (Partial<InfraConfig> & { id: string })[] = [];
for (const dbConfig of dbInfraConfigs) {
let envValue = process.env[dbConfig.name];
// lastSyncedEnvFileValue null check for backward compatibility from 2024.10.2 and below
if (!dbConfig.lastSyncedEnvFileValue && envValue) {
const configValue = dbConfig.isEncrypted ? encrypt(envValue) : envValue;
updateRequiredObjs.push({
id: dbConfig.id,
value: dbConfig.value === null ? configValue : undefined,
lastSyncedEnvFileValue: configValue,
});
continue;
}
// If the value in the database is different from the value in the .env file, means the value in the .env file has been updated
const rawLastSyncedEnvFileValue = dbConfig.isEncrypted
? decrypt(dbConfig.lastSyncedEnvFileValue)
: dbConfig.lastSyncedEnvFileValue;
if (rawLastSyncedEnvFileValue != envValue) {
const configValue = dbConfig.isEncrypted ? encrypt(envValue) : envValue;
updateRequiredObjs.push({
id: dbConfig.id,
value: configValue ?? null,
lastSyncedEnvFileValue: configValue ?? null,
});
}
}
return updateRequiredObjs;
}
/**
* Verify if 'infra_config' table is loaded with all entries
* @returns boolean
*/
export async function isInfraConfigTablePopulated(): Promise<boolean> {
try {
const propsRemainingToInsert = await getMissingInfraConfigEntries();
const defaultInfraConfigs = await getDefaultInfraConfigs();
const propsRemainingToInsert =
await getMissingInfraConfigEntries(defaultInfraConfigs);
if (propsRemainingToInsert.length > 0) {
console.log(
@@ -306,10 +385,11 @@ export function stopApp() {
}
/**
* Get the configured SSO providers
* Get the configured SSO providers from .env file
* @description This function verify if the required parameters for each SSO provider are configured in .env file. Usage on first time setup and reset.
* @returns Array of configured SSO providers
*/
export function getConfiguredSSOProviders() {
export function getConfiguredSSOProvidersFromEnvFile() {
const allowedAuthProviders: string[] =
process.env.VITE_ALLOWED_AUTH_PROVIDERS.split(',');
let configuredAuthProviders: string[] = [];
@@ -320,7 +400,6 @@ export function getConfiguredSSOProviders() {
const isConfigured = configParameters.every((configParameter) => {
return process.env[configParameter];
});
if (isConfigured) configuredAuthProviders.push(provider);
};
@@ -337,7 +416,47 @@ export function getConfiguredSSOProviders() {
console.log(
`${unConfiguredAuthProviders.join(
',',
)} SSO auth provider(s) are not configured properly. Do configure them from Admin Dashboard.`,
)} SSO auth provider(s) are not configured properly in .env file. Do configure them from Admin Dashboard.`,
);
}
return configuredAuthProviders.join(',');
}
/**
* Get the configured SSO providers from 'infra_config' table.
* @description Usage every time the app starts by AuthModule to initiate Strategies.
* @returns Array of configured SSO providers
*/
export async function getConfiguredSSOProvidersFromInfraConfig() {
const env = await loadInfraConfiguration();
const allowedAuthProviders: string[] =
env['INFRA'].VITE_ALLOWED_AUTH_PROVIDERS.split(',');
let configuredAuthProviders: string[] = [];
const addProviderIfConfigured = (provider) => {
const configParameters: string[] = AuthProviderConfigurations[provider];
const isConfigured = configParameters.every((configParameter) => {
return env['INFRA'][configParameter];
});
if (isConfigured) configuredAuthProviders.push(provider);
};
allowedAuthProviders.forEach((provider) => addProviderIfConfigured(provider));
if (configuredAuthProviders.length === 0) {
return '';
} else if (allowedAuthProviders.length !== configuredAuthProviders.length) {
const prisma = new PrismaService();
await prisma.infraConfig.update({
where: { name: InfraConfigEnum.VITE_ALLOWED_AUTH_PROVIDERS },
data: { value: configuredAuthProviders.join(',') },
});
stopApp();
console.log(
`${configuredAuthProviders.join(',')} SSO auth provider(s) are configured properly. To enable other SSO providers, configure them from Admin Dashboard.`,
);
}

View File

@@ -28,8 +28,8 @@ const dbInfraConfigs: dbInfraConfig[] = [
id: '3',
name: InfraConfigEnum.GOOGLE_CLIENT_ID,
value: 'abcdefghijkl',
lastSyncedEnvFileValue: 'abcdefghijkl',
isEncrypted: false,
active: true,
createdOn: INITIALIZED_DATE_CONST,
updatedOn: INITIALIZED_DATE_CONST,
},
@@ -37,8 +37,8 @@ const dbInfraConfigs: dbInfraConfig[] = [
id: '4',
name: InfraConfigEnum.VITE_ALLOWED_AUTH_PROVIDERS,
value: 'google',
lastSyncedEnvFileValue: 'google',
isEncrypted: false,
active: true,
createdOn: INITIALIZED_DATE_CONST,
updatedOn: INITIALIZED_DATE_CONST,
},
@@ -72,8 +72,8 @@ describe('InfraConfigService', () => {
id: '',
name,
value,
lastSyncedEnvFileValue: value,
isEncrypted: false,
active: true,
createdOn: new Date(),
updatedOn: new Date(),
});
@@ -97,8 +97,8 @@ describe('InfraConfigService', () => {
id: '',
name,
value,
lastSyncedEnvFileValue: value,
isEncrypted: false,
active: true,
createdOn: new Date(),
updatedOn: new Date(),
});
@@ -122,8 +122,8 @@ describe('InfraConfigService', () => {
id: '',
name,
value,
lastSyncedEnvFileValue: value,
isEncrypted: false,
active: true,
createdOn: new Date(),
updatedOn: new Date(),
});
@@ -173,8 +173,8 @@ describe('InfraConfigService', () => {
id: '',
name,
value,
lastSyncedEnvFileValue: value,
isEncrypted: false,
active: true,
createdOn: new Date(),
updatedOn: new Date(),
});

View File

@@ -29,6 +29,7 @@ import {
getEncryptionRequiredInfraConfigEntries,
getMissingInfraConfigEntries,
stopApp,
syncInfraConfigWithEnvFile,
} from './helper';
import { EnableAndDisableSSOArgs, InfraConfigArgs } from './input-args';
import { AuthProvider } from 'src/auth/helper';
@@ -65,8 +66,11 @@ export class InfraConfigService implements OnModuleInit {
*/
async initializeInfraConfigTable() {
try {
const defaultInfraConfigs = await getDefaultInfraConfigs();
// Adding missing InfraConfigs to the database (with encrypted values)
const propsToInsert = await getMissingInfraConfigEntries();
const propsToInsert =
await getMissingInfraConfigEntries(defaultInfraConfigs);
if (propsToInsert.length > 0) {
await this.prisma.infraConfig.createMany({ data: propsToInsert });
@@ -74,7 +78,7 @@ export class InfraConfigService implements OnModuleInit {
// Encrypting previous InfraConfigs that are required to be encrypted
const encryptionRequiredEntries =
await getEncryptionRequiredInfraConfigEntries();
await getEncryptionRequiredInfraConfigEntries(defaultInfraConfigs);
if (encryptionRequiredEntries.length > 0) {
const dbOperations = encryptionRequiredEntries.map((dbConfig) => {
@@ -87,8 +91,25 @@ export class InfraConfigService implements OnModuleInit {
await Promise.allSettled(dbOperations);
}
// Sync the InfraConfigs with the .env file, if .env file updates later on
const envFileChangesRequired = await syncInfraConfigWithEnvFile();
if (envFileChangesRequired.length > 0) {
const dbOperations = envFileChangesRequired.map((dbConfig) => {
const { id, ...dataObj } = dbConfig;
return this.prisma.infraConfig.update({
where: { id: dbConfig.id },
data: dataObj,
});
});
await Promise.allSettled(dbOperations);
}
// Restart the app if needed
if (propsToInsert.length > 0 || encryptionRequiredEntries.length > 0) {
if (
propsToInsert.length > 0 ||
encryptionRequiredEntries.length > 0 ||
envFileChangesRequired.length > 0
) {
stopApp();
}
} catch (error) {

View File

@@ -546,10 +546,12 @@ describe('ShortcodeService', () => {
);
expect(result).toEqual(<ShortcodeWithUserEmail[]>[
{
id: shortcodes[1].id,
request: JSON.stringify(shortcodes[1].request),
properties: JSON.stringify(shortcodes[1].embedProperties),
createdOn: shortcodes[1].createdOn,
id: shortcodesWithUserEmail[1].id,
request: JSON.stringify(shortcodesWithUserEmail[1].request),
properties: JSON.stringify(
shortcodesWithUserEmail[1].embedProperties,
),
createdOn: shortcodesWithUserEmail[1].createdOn,
creator: {
uid: user.uid,
email: user.email,