Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Fleet] Update secret values (API only) #156806

Merged
merged 8 commits into from
May 16, 2023
Merged

[Fleet] Update secret values (API only) #156806

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1d69b73
single update working
hop-dev May 4, 2023
f1cc822
delete unused secrets
hop-dev May 4, 2023
c6ded32
only delete if secret not in use
hop-dev May 4, 2023
590aa0f
remove comment
hop-dev May 15, 2023
fd88031
do not throw error if secrets fail to be deleted
hop-dev May 15, 2023
3be8855
add test for secret added
hop-dev May 15, 2023
c5bc8ff
make secret vars not required for better testing
hop-dev May 15, 2023
96a698a
remove duplicated tests
hop-dev May 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions x-pack/plugins/fleet/common/services/validate_package_policy.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1030,5 +1030,53 @@ describe('Fleet - validatePackagePolicyConfig', () => {

expect(res).toBeNull();
});
it('should accept a secret ref instead of a text value for a secret field', () => {
const res = validatePackagePolicyConfig(
{
value: { isSecretRef: true, id: 'secret1' },
},
{
name: 'secret_variable',
type: 'text',
secret: true,
},
'secret_variable',
safeLoad
);

expect(res).toBeNull();
});
it('secret refs should always have an id', () => {
const res = validatePackagePolicyConfig(
{
value: { isSecretRef: true },
},
{
name: 'secret_variable',
type: 'text',
secret: true,
},
'secret_variable',
safeLoad
);

expect(res).toEqual(['Secret reference is invalid, id must be a string']);
});
it('secret ref id should be a string', () => {
const res = validatePackagePolicyConfig(
{
value: { isSecretRef: true, id: 123 },
},
{
name: 'secret_variable',
type: 'text',
secret: true,
},
'secret_variable',
safeLoad
);

expect(res).toEqual(['Secret reference is invalid, id must be a string']);
});
});
});
17 changes: 17 additions & 0 deletions x-pack/plugins/fleet/common/services/validate_package_policy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,23 @@ export const validatePackagePolicyConfig = (
}
}

if (varDef.secret === true && parsedValue && parsedValue.isSecretRef === true) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Validation updated here to accept a secret reference in place of a value, e.g mongo password is a password field, but once a policy is created we only have a secret reference, so we cannot validate the field in the normal way.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See the updated unit test above for an example

if (
parsedValue.id === undefined ||
parsedValue.id === '' ||
typeof parsedValue.id !== 'string'
) {
errors.push(
i18n.translate('xpack.fleet.packagePolicyValidation.invalidSecretReference', {
defaultMessage: 'Secret reference is invalid, id must be a string',
})
);

return errors;
}
return null;
}

if (varDef.type === 'yaml') {
try {
parsedValue = safeLoadYaml(value);
Expand Down
79 changes: 66 additions & 13 deletions x-pack/plugins/fleet/server/services/package_policy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,11 @@ import { updateDatastreamExperimentalFeatures } from './epm/packages/update';
import type { PackagePolicyClient, PackagePolicyService } from './package_policy_service';
import { installAssetsForInputPackagePolicy } from './epm/packages/install';
import { auditLoggingService } from './audit_logging';
import { extractAndWriteSecrets } from './secrets';
import {
extractAndUpdateSecrets,
extractAndWriteSecrets,
deleteSecretsIfNotReferenced as deleteSecrets,
} from './secrets';

export type InputsOverride = Partial<NewPackagePolicyInput> & {
vars?: Array<NewPackagePolicyInput['vars'] & { name: string }>;
Expand Down Expand Up @@ -242,15 +246,15 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
const { secretsStorage: secretsStorageEnabled } = appContextService.getExperimentalFeatures();
if (secretsStorageEnabled) {
const secretsRes = await extractAndWriteSecrets({
packagePolicy: enrichedPackagePolicy,
packagePolicy: { ...enrichedPackagePolicy, inputs },
packageInfo: pkgInfo,
esClient,
});

enrichedPackagePolicy = secretsRes.packagePolicy;
secretReferences = secretsRes.secret_references;
secretReferences = secretsRes.secretReferences;

inputs = getInputsWithStreamIds(enrichedPackagePolicy, packagePolicyId);
inputs = enrichedPackagePolicy.inputs as PackagePolicyInput[];
}
inputs = await _compilePackagePolicyInputs(pkgInfo, enrichedPackagePolicy.vars || {}, inputs);

Expand Down Expand Up @@ -644,6 +648,8 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
});

let enrichedPackagePolicy: UpdatePackagePolicy;
let secretReferences: PolicySecretReference[] | undefined;
let secretsToDelete: PolicySecretReference[] | undefined;

try {
enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks(
Expand All @@ -661,7 +667,6 @@ class PackagePolicyClientImpl implements PackagePolicyClient {

const packagePolicy = { ...enrichedPackagePolicy, name: enrichedPackagePolicy.name.trim() };
const oldPackagePolicy = await this.get(soClient, id);
const { version, ...restOfPackagePolicy } = packagePolicy;

if (packagePolicyUpdate.is_managed && !options?.force) {
throw new PackagePolicyRestrictionRelatedError(`Cannot update package policy ${id}`);
Expand All @@ -678,6 +683,8 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
await requireUniqueName(soClient, enrichedPackagePolicy, id);
}

// eslint-disable-next-line prefer-const
let { version, ...restOfPackagePolicy } = packagePolicy;
let inputs = getInputsWithStreamIds(restOfPackagePolicy, oldPackagePolicy.id);

inputs = enforceFrozenInputs(oldPackagePolicy.inputs, inputs, options?.force);
Expand All @@ -697,12 +704,31 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
});
validatePackagePolicyOrThrow(packagePolicy, pkgInfo);

inputs = await _compilePackagePolicyInputs(pkgInfo, packagePolicy.vars || {}, inputs);
const { secretsStorage: secretsStorageEnabled } = appContextService.getExperimentalFeatures();
if (secretsStorageEnabled) {
const secretsRes = await extractAndUpdateSecrets({
oldPackagePolicy,
packagePolicyUpdate: { ...restOfPackagePolicy, inputs },
packageInfo: pkgInfo,
esClient,
});

restOfPackagePolicy = secretsRes.packagePolicyUpdate;
secretReferences = secretsRes.secretReferences;
secretsToDelete = secretsRes.secretsToDelete;
inputs = restOfPackagePolicy.inputs as PackagePolicyInput[];
}

inputs = await _compilePackagePolicyInputs(pkgInfo, restOfPackagePolicy.vars || {}, inputs);
elasticsearchPrivileges = pkgInfo.elasticsearch?.privileges;
}

// Handle component template/mappings updates for experimental features, e.g. synthetic source
await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy });
await handleExperimentalDatastreamFeatureOptIn({
soClient,
esClient,
packagePolicy: restOfPackagePolicy,
});

await soClient.update<PackagePolicySOAttributes>(
SAVED_OBJECT_TYPE,
Expand All @@ -714,6 +740,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
: {}),
inputs,
...(elasticsearchPrivileges && { elasticsearch: { privileges: elasticsearchPrivileges } }),
...(secretReferences?.length && { secret_references: secretReferences }),
revision: oldPackagePolicy.revision + 1,
updated_at: new Date().toISOString(),
updated_by: options?.user?.username ?? 'system',
Expand Down Expand Up @@ -767,7 +794,11 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
pkgName: newPolicy.package!.name,
currentVersion: newPolicy.package!.version,
});
await Promise.all([bumpPromise, assetRemovePromise]);
const deleteSecretsPromise = secretsToDelete?.length
? deleteSecrets({ esClient, soClient, ids: secretsToDelete.map((s) => s.id) })
: Promise.resolve();

await Promise.all([bumpPromise, assetRemovePromise, deleteSecretsPromise]);

sendUpdatePackagePolicyTelemetryEvent(soClient, [packagePolicyUpdate], [oldPackagePolicy]);

Expand All @@ -778,8 +809,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
packagePolicyUpdates: Array<NewPackagePolicy & { version?: string; id: string }>,
options?: { user?: AuthenticatedUser; force?: boolean },
currentVersion?: string
options?: { user?: AuthenticatedUser; force?: boolean }
): Promise<{
updatedPolicies: PackagePolicy[] | null;
failedPolicies: Array<{
Expand All @@ -804,6 +834,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
}

const packageInfos = await getPackageInfoForPackagePolicies(packagePolicyUpdates, soClient);
const allSecretsToDelete: PolicySecretReference[] = [];

const policiesToUpdate: Array<SavedObjectsBulkUpdateObject<PackagePolicySOAttributes>> = [];
const failedPolicies: Array<{
Expand All @@ -820,8 +851,11 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
throw new Error('Package policy not found');
}

let secretReferences: PolicySecretReference[] | undefined;

// id and version are not part of the saved object attributes
const { version, id: _id, ...restOfPackagePolicy } = packagePolicy;
// eslint-disable-next-line prefer-const
let { version, id: _id, ...restOfPackagePolicy } = packagePolicy;

if (packagePolicyUpdate.is_managed && !options?.force) {
throw new PackagePolicyRestrictionRelatedError(`Cannot update package policy ${id}`);
Expand All @@ -837,7 +871,21 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
);
if (pkgInfo) {
validatePackagePolicyOrThrow(packagePolicy, pkgInfo);

const { secretsStorage: secretsStorageEnabled } =
appContextService.getExperimentalFeatures();
if (secretsStorageEnabled) {
const secretsRes = await extractAndUpdateSecrets({
juliaElastic marked this conversation as resolved.
Show resolved Hide resolved
oldPackagePolicy,
packagePolicyUpdate: { ...restOfPackagePolicy, inputs },
packageInfo: pkgInfo,
esClient,
});

restOfPackagePolicy = secretsRes.packagePolicyUpdate;
secretReferences = secretsRes.secretReferences;
allSecretsToDelete.push(...secretsRes.secretsToDelete);
inputs = restOfPackagePolicy.inputs as PackagePolicyInput[];
}
inputs = await _compilePackagePolicyInputs(pkgInfo, packagePolicy.vars || {}, inputs);
elasticsearchPrivileges = pkgInfo.elasticsearch?.privileges;
}
Expand All @@ -858,6 +906,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
...(elasticsearchPrivileges && {
elasticsearch: { privileges: elasticsearchPrivileges },
}),
...(secretReferences?.length && { secret_references: secretReferences }),
revision: oldPackagePolicy.revision + 1,
updated_at: new Date().toISOString(),
updated_by: options?.user?.username ?? 'system',
Expand Down Expand Up @@ -901,7 +950,11 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
});
});

await Promise.all([bumpPromise, removeAssetPromise]);
const deleteSecretsPromise = allSecretsToDelete.length
? deleteSecrets({ esClient, soClient, ids: allSecretsToDelete.map((s) => s.id) })
: Promise.resolve();

await Promise.all([bumpPromise, removeAssetPromise, deleteSecretsPromise]);

sendUpdatePackagePolicyTelemetryEvent(soClient, packagePolicyUpdates, oldPackagePolicies);

Expand Down
Loading