add on-demand import and new retention rules

.project

@@ -1,23 +1,34 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <projectDescription>
-    <name>sanecleanup</name>
-    <comment></comment>
-    <projects>
-    </projects>
-    <buildSpec>
-        <buildCommand>
-            <name>org.eclipse.jdt.core.javabuilder</name>
-            <arguments>
-            </arguments>
-        </buildCommand>
-        <buildCommand>
-            <name>org.springframework.ide.eclipse.core.springbuilder</name>
-            <arguments>
-            </arguments>
-        </buildCommand>
-    </buildSpec>
-    <natures>
-        <nature>org.springframework.ide.eclipse.core.springnature</nature>
-        <nature>org.eclipse.jdt.core.javanature</nature>
-    </natures>
-</projectDescription>
+	<name>sanecleanup</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.springframework.ide.eclipse.core.springbuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.springframework.ide.eclipse.core.springnature</nature>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+	<filteredResources>
+		<filter>
+			<id>1606724818477</id>
+			<name></name>
+			<type>30</type>
+			<matcher>
+				<id>org.eclipse.core.resources.regexFilterMatcher</id>
+				<arguments>node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
+			</matcher>
+		</filter>
+	</filteredResources>
+</projectDescription>

CHANGELOG.md

@@ -0,0 +1,52 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [2.0.0] - 2021-03-23
+
+### Changed
+
+- Cleanup jobs / retention rules are now imported on-demand based on the extensions the project uses.
+
+### Added
+
+- Custom retention cronjob to replace CMS Version [Garbage Collection][versiongc].\
+  The ootb garbage collection mechanism is over-engineered and should be a regular cronjob.
+
+- Retention rules and jobs for the promotion engine, based on [Top 10 Recommendations for Improving the Performance of your Commerce Cloud Promotion Engine][top10]
+
+[versiongc]: https://help.sap.com/viewer/9d346683b0084da2938be8a285c0c27a/2011/en-US/9089116335ac4f4d8708e0c5516531e3.html
+[top10]: https://www.sap.com/cxworks/article/538808299/top_10_recommendations_for_improving_the_performance_of_your_commerce_cloud_promotion_engine
+
+## [1.0.1] - 22020-12-09
+
+### Added
+
+- Bulk cleanup cronjob for log files - useful for a one-time cleanup before the retention 
+  job for job logs is enabled
+
+## [1.0.0] - 2020-11-26
+
+Initial release
+
+### Added
+
+- Cleanup for:
+
+    - CronJobs
+    - CronJob Histories
+    - Lob Logs / Log Files
+    - Impex Media
+    - HTTP Sessions
+    - Business Processes
+    - Carts
+
+[Unreleased]: https://github.com/sap-commerce-tools/sanecleanup/compare/v2.0.0...HEAD
+[2.0.0]: https://github.com/sap-commerce-tools/sanecleanup/compare/v1.0.1...v2.0.0
+[1.0.1]: https://github.com/sap-commerce-tools/sanecleanup/compare/v1.0.0...v1.0.1
+[1.0.0]: https://github.com/sap-commerce-tools/sanecleanup/releases/tag/v1.0.0
+

README.md

@@ -1,29 +1,33 @@
 # sanecleanup
 
-![SAP Commerce 1811+](https://img.shields.io/badge/Commerce-1811+-0051ab?logo=SAP)
+![SAP Commerce 1811+](https://img.shields.io/badge/Commerce-2005+-0051ab?logo=SAP)
 
 Sensible defaults for data retention and cleanup for SAP Commerce, based on my CX Works article [Data Maintenance and Cleanup][article]
 
 
 
-1. Download the repository as zip file
+1. Download the latest release
 1. Unpack to `hybris/bin/custom`
-1. **Review and adapt the retention rules** and cronjobs defined in `sanecleanup/resources/impex/*.impex`\
-1. If possible, disable storing of saved values / change history too! ([help.sap.com][stored], further recommendations in my [article][stored-kill])
+1. If possible, disable saved values / change history (ref. [help.sap.com][stored], further recommendations in my [article][stored-kill])
 1. Add extension to your `localextensions.xml`
 
     ````xml
    <extension name="sanecleanup" />
     ````
 
 1. Build and deploy.\
-  (The rules will be imported during system update)
+  (The rules will be automatically imported during system update)
 
 **Warning**\
-The first run of `cronJobLogCleanupCronJob` will take a _very_ long time, if you have never removed any cronjob log files (type `LogFile`).\
+The very first execution of the retention cron jobs will take a while, depending on how long your poject
+is already live and if you have cleaned up anything in the past.
+
+Consider performing a [one-time cleanup][one] before adding the extension / enabling the retention rules.
+
+Especially the first run of `cronJobLogCleanupCronJob` will take a _very_ long time, if you have never removed any cronjob log files (type `LogFile`).\
 Please consider importing and executing the script job defined in [bulkdelete-cronjoblogs.impex](resources/impex/bulkdelete-cronjoblogs.impex) **before** you set up the automated cleanup!\
-The job will remove all log files except the five most recent ones per CronJob.
-(Disclaimer: the script was tested on MS SQL / Azure SQL. It is not guaranteed to work for other Databases)
+The job will remove all log files except the five most recent logs per CronJob.
+(Disclaimer: the script was tested on MS SQL / Azure SQL. It is not guaranteed to work for other databases)
 
 ## Support 
 

extensioninfo.xml

@@ -4,6 +4,8 @@
   <extension abstractclassprefix="Generated" classprefix="Sanecleanup" jaloLogicFree="true"
              managername="SanecleanupManager" managersuperclass="de.hybris.platform.jalo.extension.Extension"
              name="sanecleanup" usemaven="false">
+    <requires-extension name="commerceservices" />
+    <requires-extension name="cms2" />
     <coremodule generated="true" manager="de.hybris.platform.jalo.extension.GenericManager"
                 packageroot="mpern.sap.cleanup"/>
   </extension>

project.properties

@@ -1,3 +1,9 @@
 sanecleanup.application-context=sanecleanup-spring.xml
 
+# disable change history entries / "Last Changes"
 # hmc.storing.modifiedvalues.size=0
+
+# replaced with mpern.sap.cleanup.cms2.CMSVersionGCPerformable
+version.gc.enabled=false
+
+sanecleanup.jobs.sessionlanguage=en

resources/impex/bulkdelete-cronjoblogs.impex

@@ -7,6 +7,7 @@ import de.hybris.platform.cronjob.enums.CronJobStatus
 import de.hybris.platform.cronjob.enums.CronJobResult
 
 // all log files EXCEPT the five most recent logs per cronjob
+// warning: query uses MS SQL dialact for partioning the logs per cronjob
 def QUERY = '''
 SELECT t.pk
 FROM

resources/impex/essentialdata-jobs.impex

@@ -0,0 +1,3 @@
+
+INSERT_UPDATE ServicelayerJob;code[unique=true];springId;
+;cmsVersionGCJob;cmsVersionGCPerformable;

resources/impex/sanecleanup/cms2/001-optimized-versiongc.impex

@@ -0,0 +1,11 @@
+
+# Import config properties into impex macros
+UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true]
+$sessionLanguage=$config-sanecleanup.jobs.sessionlanguage
+$cronExpression=$config-version.gc.cron
+
+INSERT_UPDATE CronJob;code[unique=true];job(code);sessionLanguage(isoCode)[default = $sessionLanguage]
+;cmsVersionGCCronJob;cmsVersionGCJob;
+
+INSERT_UPDATE Trigger;cronjob(code)[unique=true];cronExpression
+;cmsVersionGCCronJob; $cronExpression

resources/impex/sanecleanup/commerceservices/001-cleanup-oldcarts.impex

@@ -0,0 +1,36 @@
+# based on de.hybris.platform.commercewebservices.core.cronjob.OldCartRemovalJob
+# and de.hybris.platform.commerceservices.order.dao.impl.DefaultCommerceCartDao.getCartsForRemovalForSiteAndUser
+#
+# - cleanup anonymous carts after two weeks for *every* site (excluding saved carts)
+# - cleanup carts of registered users after four weeks for *every* site (excluding saved carts)
+
+# Import config properties into impex macros
+UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true]
+$sessionLanguage=$config-sanecleanup.jobs.sessionlanguage
+
+$twoWeeks = 1209600
+$fourWeeks = 2419200
+INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference          ;
+                                  ; cartRule           ; "
+  SELECT {c:pk}, {c:itemtype}
+    FROM { Cart AS c LEFT JOIN User AS u ON {c:user} = {u:pk} }
+   WHERE {c:saveTime} IS NULL
+     AND {u:uid} <> 'anonymous'
+     AND {c:modifiedTime} < ?CALC_RETIREMENT_TIME"                        ; $fourWeeks          ; basicRemoveCleanupAction ;
+                                  ; anonymousCartRule  ; "
+  SELECT {c:pk}, {c:itemtype}
+    FROM {Cart AS c LEFT JOIN User AS u ON {c:user} = {u:pk}}
+   WHERE {c:saveTime} IS NULL
+     AND ( {u:uid} = 'anonymous' OR {u:uid} IS NULL )
+     AND {c:modifiedTime} < ?CALC_RETIREMENT_TIME"                        ; $twoWeeks           ; basicRemoveCleanupAction ;
+
+INSERT_UPDATE RetentionJob; code[unique = true]     ; retentionRule(code); batchSize
+                   ; cartCleanupJob          ; cartRule           ; 1000
+                   ; anonymousCartCleanupJob ; anonymousCartRule  ; 1000
+INSERT_UPDATE CronJob; code[unique = true]         ; job(code)               ; sessionLanguage(isoCode)[default = $sessionLanguage]
+              ; cartCleanupCronJob          ; cartCleanupJob          ;
+              ; anonymousCartCleanupCronJob ; anonymousCartCleanupJob ;
+INSERT_UPDATE Trigger; cronJob(code)[unique = true]; cronExpression
+# every day at midnight
+              ; cartCleanupCronJob          ; 0 0 0 * * ?
+              ; anonymousCartCleanupCronJob ; 0 0 0 * * ?

resources/impex/sanecleanup/core/001-cleanup-httpsession.impex

@@ -0,0 +1,18 @@
+# delete all stored sessions one day after there last update
+
+# Import config properties into impex macros
+UPDATE GenericItem[processor = de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor]; pk[unique = true]
+$sessionLanguage = $config-sanecleanup.jobs.sessionlanguage
+
+$oneDay = 86400
+INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]   ; searchQuery; retentionTimeSeconds; actionReference          ;
+                                  ; storedHttpSessionRule ; "select {s:pk}, {s:itemtype}
+   from {StoredHttpSession as s}
+   where {s:modifiedTime} < ?CALC_RETIREMENT_TIME"                     ; $oneDay             ; basicRemoveCleanupAction ;
+INSERT_UPDATE RetentionJob; code[unique = true]         ; retentionRule(code)   ; batchSize
+                   ; storedHttpSessionCleanupJob ; storedHttpSessionRule ; 1000
+INSERT_UPDATE CronJob; code[unique = true]             ; job(code)                   ; sessionLanguage(isoCode)[default = $sessionLanguage]
+              ; storedHttpSessionCleanupCronJob ; storedHttpSessionCleanupJob ;
+INSERT_UPDATE Trigger; cronJob(code)[unique = true]    ; cronExpression
+# every 30 minutes
+              ; storedHttpSessionCleanupCronJob ; 0 0/30 * * * ?

resources/impex/sanecleanup/impex/001_cleanup-impex.impex

@@ -0,0 +1,14 @@
+$twoWeeks = 1209600
+INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference          ;
+                                  ; impexMediaRule     ; "
+  SELECT {i:pk}, {i:itemtype}
+    FROM {ImpexMedia AS i}
+   WHERE {i:code} LIKE '00______'
+     AND {i:modifiedTime} < ?CALC_RETIREMENT_TIME"                        ; $twoWeeks           ; basicRemoveCleanupAction ;
+INSERT_UPDATE RetentionJob; code[unique = true]  ; retentionRule(code); batchSize
+                   ; impexMediaCleanupJob ; impexMediaRule     ; 1000
+INSERT_UPDATE CronJob; code[unique = true]      ; job(code)            ; sessionLanguage(isoCode)[default = en]
+              ; impexMediaCleanupCronJob ; impexMediaCleanupJob ;
+INSERT_UPDATE Trigger; cronJob(code)[unique = true]; cronExpression
+# every day at midnight
+              ; impexMediaCleanupCronJob    ; 0 0 0 * * ?