From a64bbaef920048b7ece0ab3441718dd417330251 Mon Sep 17 00:00:00 2001 From: Sumeng Wang Date: Fri, 21 Jul 2023 09:58:33 -0700 Subject: [PATCH] fix pushdown not exported export should get pushdown --- .../ConfigModelessActionButtons/index.tsx | 10 ++--- .../PushdownTabContent.tsx | 26 ++----------- .../Store/ActionCreator.js | 11 +++++- .../PipelineDetailsActionsButton/index.tsx | 35 ++++++++++++------ app/cdap/services/helpers.js | 37 ++++++++++++++++++- 5 files changed, 78 insertions(+), 41 deletions(-) diff --git a/app/cdap/components/PipelineConfigurations/ConfigurationsContent/ConfigModelessActionButtons/index.tsx b/app/cdap/components/PipelineConfigurations/ConfigurationsContent/ConfigModelessActionButtons/index.tsx index 17a675a0a42..27eb82ad8ae 100644 --- a/app/cdap/components/PipelineConfigurations/ConfigurationsContent/ConfigModelessActionButtons/index.tsx +++ b/app/cdap/components/PipelineConfigurations/ConfigurationsContent/ConfigModelessActionButtons/index.tsx @@ -87,12 +87,10 @@ export const ConfigModelessActionButtons = ({ ...props }: IConfigModelessActionB const saveAndAction = (actionFn) => { setSaveLoading(true); - let observable; - if (lifecycleManagementEditEnabled) { - observable = updatePreferences(); - } else { - observable = Observable.forkJoin(updatePipeline(), updatePreferences()); - } + const observable = Observable.forkJoin( + updatePipeline(lifecycleManagementEditEnabled), + updatePreferences() + ); observable.subscribe( () => { actionFn(); diff --git a/app/cdap/components/PipelineConfigurations/ConfigurationsContent/PushdownTabContent.tsx b/app/cdap/components/PipelineConfigurations/ConfigurationsContent/PushdownTabContent.tsx index f15af6c7ed6..cedf210e0ae 100644 --- a/app/cdap/components/PipelineConfigurations/ConfigurationsContent/PushdownTabContent.tsx +++ b/app/cdap/components/PipelineConfigurations/ConfigurationsContent/PushdownTabContent.tsx @@ -26,35 +26,17 @@ import { convertKeyValuePairsToMap, convertMapToKeyValuePairs, flattenObj, - unflatternStringToObj, + getPushdownObjectFromRuntimeArgs, + isPushdownEnabled, } from 'services/helpers'; import { useFeatureFlagDefaultFalse } from 'services/react/customHooks/useFeatureFlag'; const getPushdownEnabledValue = (state) => { - const pushdownEnabledKeyValuePair = state.runtimeArgs.pairs.find( - (pair) => pair.key === GENERATED_RUNTIMEARGS.PIPELINE_PUSHDOWN_ENABLED - ); - return pushdownEnabledKeyValuePair - ? pushdownEnabledKeyValuePair.value === 'true' - : state.pushdownEnabled; + return isPushdownEnabled(state.runtimeArgs) || state.pushdownEnabled; }; const getTransformationPushdownValue = (state) => { - if (state.transformationPushdown) { - return state.transformationPushdown; - } - const transformationPushdownKeyValuePair = state.runtimeArgs.pairs.filter((pair) => - pair.key.startsWith(GENERATED_RUNTIMEARGS.PIPELINE_TRANSFORMATION_PUSHDOWN_PREFIX) - ); - const pushdownObject = {}; - transformationPushdownKeyValuePair.forEach((pair) => { - // unflattern the runtimeargs key and put it in pushdown object - const trimmedKey = pair.key.substring( - GENERATED_RUNTIMEARGS.PIPELINE_TRANSFORMATION_PUSHDOWN_PREFIX.length - ); - unflatternStringToObj(pushdownObject, trimmedKey, pair.value); - }); - return pushdownObject; + return getPushdownObjectFromRuntimeArgs(state.runtimeArgs) || state.transformationPushdown; }; export default function PushdownTabContent({}) { diff --git a/app/cdap/components/PipelineConfigurations/Store/ActionCreator.js b/app/cdap/components/PipelineConfigurations/Store/ActionCreator.js index d59a2f53217..2c33fd504b6 100644 --- a/app/cdap/components/PipelineConfigurations/Store/ActionCreator.js +++ b/app/cdap/components/PipelineConfigurations/Store/ActionCreator.js @@ -44,6 +44,7 @@ import uniqBy from 'lodash/uniqBy'; import cloneDeep from 'lodash/cloneDeep'; import { CLOUD } from 'services/global-constants'; import { Observable } from 'rxjs/Observable'; +import { of } from 'rxjs/observable/of'; import { map } from 'rxjs/operators'; // Filter certain preferences from being shown in the run time arguments @@ -150,7 +151,7 @@ const updatePreferences = () => { ); }; -const updatePipeline = () => { +const updatePipeline = (lifecycleManagementEditEnabled = true) => { let detailStoreState = PipelineDetailStore.getState(); let { name, description, artifact, principal } = detailStoreState; let { stages, connections, comments } = detailStoreState.config; @@ -225,6 +226,14 @@ const updatePipeline = () => { config = { ...commonConfig, ...sqlOnlyConfig }; } + if (lifecycleManagementEditEnabled) { + PipelineDetailStore.dispatch({ + type: PipelineDetailActions.SET_CONFIG, + payload: { config }, + }); + return of({}); + } + let publishObservable = MyPipelineApi.publish( { namespace: getCurrentNamespace(), diff --git a/app/cdap/components/PipelineDetails/PipelineDetailsTopPanel/PipelineDetailsDetailsActions/PipelineDetailsActionsButton/index.tsx b/app/cdap/components/PipelineDetails/PipelineDetailsTopPanel/PipelineDetailsDetailsActions/PipelineDetailsActionsButton/index.tsx index f2473cc3b6e..c2b278527d6 100644 --- a/app/cdap/components/PipelineDetails/PipelineDetailsTopPanel/PipelineDetailsDetailsActions/PipelineDetailsActionsButton/index.tsx +++ b/app/cdap/components/PipelineDetails/PipelineDetailsTopPanel/PipelineDetailsDetailsActions/PipelineDetailsActionsButton/index.tsx @@ -28,7 +28,11 @@ import classnames from 'classnames'; import { duplicatePipeline, editPipeline } from 'services/PipelineUtils'; import cloneDeep from 'lodash/cloneDeep'; import downloadFile from 'services/download-file'; -import { santizeStringForHTMLID } from 'services/helpers'; +import { + getPushdownObjectFromRuntimeArgs, + isPushdownEnabled, + santizeStringForHTMLID, +} from 'services/helpers'; import { deleteEditDraft } from 'components/PipelineList/DeployedPipelineView/store/ActionCreator'; import { DiscardDraftModal } from 'components/shared/DiscardDraftModal'; import { CommitModal } from 'components/SourceControlManagement/LocalPipelineListView/CommitModal'; @@ -40,6 +44,7 @@ import { setPullStatus, setSourceControlMeta, } from 'components/PipelineDetails/store/ActionCreator'; +import PipelineConfigurationsStore from 'components/PipelineConfigurations/Store'; require('./PipelineDetailsActionsButton.scss'); const PREFIX = 'features.PipelineDetails.TopPanel'; @@ -123,16 +128,24 @@ class PipelineDetailsActionsButton extends Component { + const pipelineConfig = { + name: this.props.pipelineName, + description: this.props.description, + artifact: this.props.artifact, + config: cloneDeep(this.props.config), // currently doing a cloneDeep because angular is mutating this state... + version: this.props.version, + }; + const { runtimeArgs } = PipelineConfigurationsStore.getState(); + pipelineConfig.config.pushdownEnabled = isPushdownEnabled(runtimeArgs); + pipelineConfig.config.transformationPushdown = getPushdownObjectFromRuntimeArgs(runtimeArgs); + return pipelineConfig; }; + public pipelineConfig = this.getPipelineConfig(); + public duplicateConfigAndNavigate = () => { - duplicatePipeline(this.props.pipelineName, sanitizeConfig(this.pipelineConfig)); + duplicatePipeline(this.props.pipelineName, sanitizeConfig(this.getPipelineConfig())); }; public toggleDiscardConfirmation = () => { @@ -151,7 +164,7 @@ class PipelineDetailsActionsButton extends Component { this.setState({ showPopover: false }); }; - downloadFile(this.pipelineConfig, closePopoverCb); + downloadFile(this.getPipelineConfig(), closePopoverCb); }; public toggleExportModal = () => { @@ -241,7 +254,7 @@ class PipelineDetailsActionsButton extends Component ); diff --git a/app/cdap/services/helpers.js b/app/cdap/services/helpers.js index 3ca3c60ae8f..18d7892a594 100644 --- a/app/cdap/services/helpers.js +++ b/app/cdap/services/helpers.js @@ -31,6 +31,7 @@ import isArray from 'lodash/isArray'; // We don't use webpack alias here because this is used in Footer which is used in login app // And for login 'components/Lab/..' aliases to components folder inside login app. import experimentsList from '../components/Lab/experiment-list.tsx'; +import { GENERATED_RUNTIMEARGS } from './global-constants.js'; /* Purpose: Query a json object or an array of json objects Return: Returns undefined if property is not defined(never set) and @@ -897,6 +898,38 @@ const PIPELINE_ARTIFACTS = [ 'cdap-data-streams', ]; +/** + * + * @param {object} runtimeArgs + * @returns a pushdown config object generated from runtimeArgs + */ +const getPushdownObjectFromRuntimeArgs = (runtimeArgs) => { + const transformationPushdownKeyValuePair = runtimeArgs.pairs.filter((pair) => + pair.key.startsWith(GENERATED_RUNTIMEARGS.PIPELINE_TRANSFORMATION_PUSHDOWN_PREFIX) + ); + const pushdownObject = {}; + transformationPushdownKeyValuePair.forEach((pair) => { + // unflattern the runtimeargs key and put it in pushdown object + const trimmedKey = pair.key.substring( + GENERATED_RUNTIMEARGS.PIPELINE_TRANSFORMATION_PUSHDOWN_PREFIX.length + ); + unflatternStringToObj(pushdownObject, trimmedKey, pair.value); + }); + return pushdownObject +} + +/** + * + * @param {object} runtimeArgs + * @returns a boolean value if pushdown enabled + */ +const isPushdownEnabled = (runtimeArgs) => { + const pushdownEnabledKeyValuePair = runtimeArgs.pairs.find( + (pair) => pair.key === GENERATED_RUNTIMEARGS.PIPELINE_PUSHDOWN_ENABLED + ); + return pushdownEnabledKeyValuePair ? pushdownEnabledKeyValuePair.value === 'true' : false +} + export { openLinkInNewTab, objectQuery, @@ -959,5 +992,7 @@ export { unflatternStringToObj, arrayOfStringsMatchTargetPrefix, PIPELINE_ARTIFACTS, - BATCH_PIPELINE_TYPE + BATCH_PIPELINE_TYPE, + getPushdownObjectFromRuntimeArgs, + isPushdownEnabled };