diff --git a/docs/_configuration/event-stream.md b/docs/_configuration/event-stream.md index 044144c488..6077f949fd 100644 --- a/docs/_configuration/event-stream.md +++ b/docs/_configuration/event-stream.md @@ -152,9 +152,15 @@ policy of a few seconds can be set on the event source of the Event Stream. ````turtle @prefix ldes: . +@prefix server: . +@prefix genericES: . + +server:generic-eventstream a ldes:EventStream ; + ldes:eventSource genericES:eventSource . -ldes:retentionPolicy [ +genericES:eventSource a ldes:EventSource ; + ldes:retentionPolicy [ a ldes:LatestVersionSubset ; ldes:amount 0 ; - ] ; + ] . ```` \ No newline at end of file diff --git a/docs/_configuration/fragmentations/reference.md b/docs/_configuration/fragmentations/reference.md index ce21fef273..72c04248fb 100644 --- a/docs/_configuration/fragmentations/reference.md +++ b/docs/_configuration/fragmentations/reference.md @@ -48,7 +48,7 @@ Example properties: tree:fragmentationStrategy [ a tree:ReferenceFragmentation ; - tree:fragmentationPath "" ; + tree:fragmentationPath ; tree:fragmentationKey "version" ; ] . ``` @@ -84,6 +84,6 @@ The selected object would be `` After ingestion the member will be part of the following fragment -- http://localhost:8080/addresses/by-version?version= +- http://localhost:8080/addresses/by-version?version=http%3A%2F%2Fnjh.me%2Foriginal-id%2F123 NOTE: "version" is based on the configuration property "tree:fragmentationKey". \ No newline at end of file diff --git a/docs/how-to-run.md b/docs/how-to-run.md index 2a80d54f65..8826916cd3 100644 --- a/docs/how-to-run.md +++ b/docs/how-to-run.md @@ -136,7 +136,7 @@ Here is an explanation provided for all the possibilities on how to tweak and co ldes-server.compaction-duration Defines how long the redundant compacted fragments will remain on the server No - PD7 + P7D Maintenance diff --git a/ldes-server-admin/ldes-server-admin-common/pom.xml b/ldes-server-admin/ldes-server-admin-common/pom.xml index 366e4adc80..56719b8297 100644 --- a/ldes-server-admin/ldes-server-admin-common/pom.xml +++ b/ldes-server-admin/ldes-server-admin-common/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-admin - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-admin-common diff --git a/ldes-server-admin/ldes-server-admin-rest/pom.xml b/ldes-server-admin/ldes-server-admin-rest/pom.xml index 008a12e88b..3279053084 100644 --- a/ldes-server-admin/ldes-server-admin-rest/pom.xml +++ b/ldes-server-admin/ldes-server-admin-rest/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-admin - 3.6.0 + 3.6.1-SNAPSHOT diff --git a/ldes-server-admin/ldes-server-admin-rest/src/main/resources/viewShaclShape.ttl b/ldes-server-admin/ldes-server-admin-rest/src/main/resources/viewShaclShape.ttl index 2abc201a00..0eacd3107d 100644 --- a/ldes-server-admin/ldes-server-admin-rest/src/main/resources/viewShaclShape.ttl +++ b/ldes-server-admin/ldes-server-admin-rest/src/main/resources/viewShaclShape.ttl @@ -123,7 +123,7 @@ tree:FragmentationKey sh:name "Fragmentation Path"; sh:description "Defines the request parameter that will be used in the uri"; sh:path tree:fragmentationKey ; - sh:nodeKind sh:IRI ; + sh:datatype xsd:string ; sh:maxCount 1 . ldes:TimeBasedRetentionPolicy @@ -144,18 +144,18 @@ ldes:VersionBasedRetentionPolicy sh:targetClass ldes:LatestVersionSubset ; sh:class ldes:LatestVersionSubset ; sh:zeroOrOnePath true; - sh:property ldes:VersionbaserRetentionAmount . + sh:property ldes:VersionbasedRetentionAmount . tree:TimebasedRetentionValue a sh:PropertyShape; sh:name "Value"; sh:description "The duration during which members will be kept for a certain retention policy."; sh:path tree:value ; - sh:datatype ; + sh:datatype xsd:duration ; sh:minCount 1 ; sh:maxCount 1 . -ldes:VersionbaserRetentionAmount +ldes:VersionbasedRetentionAmount a sh:PropertyShape; sh:name "Amount"; sh:description "The duration during which members will be kept for a certain retention policy."; diff --git a/ldes-server-admin/pom.xml b/ldes-server-admin/pom.xml index 077d21aa6f..6df15b5590 100644 --- a/ldes-server-admin/pom.xml +++ b/ldes-server-admin/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.6.0 + 3.6.1-SNAPSHOT pom diff --git a/ldes-server-application/pom.xml b/ldes-server-application/pom.xml index 877d0d5ff6..de795e7566 100644 --- a/ldes-server-application/pom.xml +++ b/ldes-server-application/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-application diff --git a/ldes-server-domain/pom.xml b/ldes-server-domain/pom.xml index ac331dabd0..60464a014c 100644 --- a/ldes-server-domain/pom.xml +++ b/ldes-server-domain/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-domain diff --git a/ldes-server-fetch/ldes-server-fetch-common/pom.xml b/ldes-server-fetch/ldes-server-fetch-common/pom.xml index 7759848c9c..795132eb85 100644 --- a/ldes-server-fetch/ldes-server-fetch-common/pom.xml +++ b/ldes-server-fetch/ldes-server-fetch-common/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-fetch - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-fetch-common diff --git a/ldes-server-fetch/ldes-server-fetch-rest/pom.xml b/ldes-server-fetch/ldes-server-fetch-rest/pom.xml index 3f783d082e..5490e06db3 100644 --- a/ldes-server-fetch/ldes-server-fetch-rest/pom.xml +++ b/ldes-server-fetch/ldes-server-fetch-rest/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-fetch - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-fetch-rest diff --git a/ldes-server-fetch/pom.xml b/ldes-server-fetch/pom.xml index 4bb7c61202..263708cb2e 100644 --- a/ldes-server-fetch/pom.xml +++ b/ldes-server-fetch/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.6.0 + 3.6.1-SNAPSHOT pom ldes-server-fetch diff --git a/ldes-server-fragmentation/ldes-server-fragmentation-common/pom.xml b/ldes-server-fragmentation/ldes-server-fragmentation-common/pom.xml index 954f870bd9..773f57d8ec 100644 --- a/ldes-server-fragmentation/ldes-server-fragmentation-common/pom.xml +++ b/ldes-server-fragmentation/ldes-server-fragmentation-common/pom.xml @@ -5,7 +5,7 @@ ldes-server-fragmentation be.vlaanderen.informatievlaanderen.vsds - 3.6.0 + 3.6.1-SNAPSHOT 4.0.0 diff --git a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootBucketCreatorImpl.java b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootBucketCreatorImpl.java index 272843a1de..c3fd3907cb 100644 --- a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootBucketCreatorImpl.java +++ b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootBucketCreatorImpl.java @@ -16,7 +16,7 @@ public RootBucketCreatorImpl(BucketRepository bucketRepository) { @Override public void createRootBucketForView(ViewName viewName) { if (bucketRepository.retrieveRootBucket(viewName).isEmpty()) { - bucketRepository.insertBucket(Bucket.createRootBucketForView(viewName)); + bucketRepository.insertRootBucket(Bucket.createRootBucketForView(viewName)); } } } diff --git a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java index b2adaec89e..ed400e58d4 100644 --- a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java +++ b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java @@ -6,6 +6,6 @@ import java.util.Optional; public interface BucketRepository { - Bucket insertBucket(Bucket bucket); + Bucket insertRootBucket(Bucket bucket); Optional retrieveRootBucket(ViewName viewName); } diff --git a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketCreatorImplTest.java b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketCreatorImplTest.java index d8cde4cf07..ebee82945f 100644 --- a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketCreatorImplTest.java +++ b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketCreatorImplTest.java @@ -29,7 +29,7 @@ void when_RootFragmentDoesNotExist_ItIsCreatedAndSaved() { InOrder inOrder = inOrder(bucketRepository); inOrder.verify(bucketRepository).retrieveRootBucket(VIEW_NAME); - inOrder.verify(bucketRepository).insertBucket(Bucket.createRootBucketForView(VIEW_NAME)); + inOrder.verify(bucketRepository).insertRootBucket(Bucket.createRootBucketForView(VIEW_NAME)); inOrder.verifyNoMoreInteractions(); } diff --git a/ldes-server-fragmentation/ldes-server-fragmentation-geospatial/pom.xml b/ldes-server-fragmentation/ldes-server-fragmentation-geospatial/pom.xml index 5f5df05a63..6b0af1865c 100644 --- a/ldes-server-fragmentation/ldes-server-fragmentation-geospatial/pom.xml +++ b/ldes-server-fragmentation/ldes-server-fragmentation-geospatial/pom.xml @@ -3,7 +3,7 @@ ldes-server-fragmentation be.vlaanderen.informatievlaanderen.vsds - 3.6.0 + 3.6.1-SNAPSHOT 4.0.0 diff --git a/ldes-server-fragmentation/ldes-server-fragmentation-reference/pom.xml b/ldes-server-fragmentation/ldes-server-fragmentation-reference/pom.xml index 382bb2bd8d..fed11a7d99 100644 --- a/ldes-server-fragmentation/ldes-server-fragmentation-reference/pom.xml +++ b/ldes-server-fragmentation/ldes-server-fragmentation-reference/pom.xml @@ -5,7 +5,7 @@ ldes-server-fragmentation be.vlaanderen.informatievlaanderen.vsds - 3.6.0 + 3.6.1-SNAPSHOT 4.0.0 jar diff --git a/ldes-server-fragmentation/ldes-server-fragmentation-timebased-hierarchical/pom.xml b/ldes-server-fragmentation/ldes-server-fragmentation-timebased-hierarchical/pom.xml index a24cd99a1d..3ab2936e72 100644 --- a/ldes-server-fragmentation/ldes-server-fragmentation-timebased-hierarchical/pom.xml +++ b/ldes-server-fragmentation/ldes-server-fragmentation-timebased-hierarchical/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-fragmentation - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-fragmentation-timebased-hierarchical diff --git a/ldes-server-fragmentation/ldes-server-pagination/pom.xml b/ldes-server-fragmentation/ldes-server-pagination/pom.xml index 60ca224c84..0ed0cf8dab 100644 --- a/ldes-server-fragmentation/ldes-server-pagination/pom.xml +++ b/ldes-server-fragmentation/ldes-server-pagination/pom.xml @@ -6,7 +6,7 @@ ldes-server-fragmentation be.vlaanderen.informatievlaanderen.vsds - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-pagination diff --git a/ldes-server-fragmentation/pom.xml b/ldes-server-fragmentation/pom.xml index d05c9b0436..13de1a2b0e 100644 --- a/ldes-server-fragmentation/pom.xml +++ b/ldes-server-fragmentation/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-fragmentation diff --git a/ldes-server-infra-postgres/pom.xml b/ldes-server-infra-postgres/pom.xml index 3596389598..48c2d1d796 100644 --- a/ldes-server-infra-postgres/pom.xml +++ b/ldes-server-infra-postgres/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.6.0 + 3.6.1-SNAPSHOT pom @@ -24,7 +24,7 @@ 42.7.4 - 3.7.4 + 3.9.0 diff --git a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml index fedc4f8b10..6a541e4306 100644 --- a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.6.0 + 3.6.1-SNAPSHOT postgres-admin-repository diff --git a/ldes-server-infra-postgres/postgres-fetch-repository/pom.xml b/ldes-server-infra-postgres/postgres-fetch-repository/pom.xml index ca91af5731..a8d6c8ce82 100644 --- a/ldes-server-infra-postgres/postgres-fetch-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-fetch-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.6.0 + 3.6.1-SNAPSHOT postgres-fetch-repository diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml index 483deb5766..f4d402b6b0 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.6.0 + 3.6.1-SNAPSHOT postgres-fragmentation-repository diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java index b9edd5f64b..3a80d763a5 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java @@ -31,10 +31,10 @@ public BucketPostgresRepository(ViewEntityRepository viewEntityRepository, Bucke @Override @Transactional - public Bucket insertBucket(Bucket bucket) { + public Bucket insertRootBucket(Bucket bucket) { String sql = """ - INSERT INTO pages (bucket_id, expiration, partial_url) - VALUES (:bucketId, NULL, :partialUrl) + INSERT INTO pages (bucket_id, expiration, partial_url, is_root) + VALUES (:bucketId, NULL, :partialUrl, true) ON CONFLICT DO NOTHING """; ViewEntity view = viewEntityRepository.findByViewName(bucket.getViewName().getCollectionName(), bucket.getViewName().getViewName()) diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java index 2ad37cf081..e36c07155c 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java @@ -41,7 +41,7 @@ void test_Insertion() { final Bucket bucketToSave = new Bucket(BucketDescriptor.of(new BucketDescriptorPair("key", "value"), new BucketDescriptorPair("k", "v")), VIEW_NAME); final Bucket expectedSavedBucket = new Bucket(1L, BucketDescriptor.of(new BucketDescriptorPair("key", "value"), new BucketDescriptorPair("k", "v")), VIEW_NAME, List.of(), 0); - final Bucket result = bucketPostgresRepository.insertBucket(bucketToSave); + final Bucket result = bucketPostgresRepository.insertRootBucket(bucketToSave); assertThat(result) .usingRecursiveComparison() diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml index df70039ee0..45401bfef5 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.6.0 + 3.6.1-SNAPSHOT postgres-ingest-repository diff --git a/ldes-server-infra-postgres/postgres-liquibase/pom.xml b/ldes-server-infra-postgres/postgres-liquibase/pom.xml index 0d09fad72c..9a728b0936 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/pom.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.6.0 + 3.6.1-SNAPSHOT postgres-liquibase diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_column.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_column.xml new file mode 100644 index 0000000000..63f4206079 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_column.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_values.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_values.sql new file mode 100644 index 0000000000..d0836c3fd8 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_values.sql @@ -0,0 +1,7 @@ +WITH root_page_partial_urls AS ( + SELECT CONCAT('/', c.name, '/', v.name) AS partial_url + FROM collections c + JOIN views v ON c.collection_id = v.collection_id +) +UPDATE pages SET is_root = true, immutable = false +WHERE partial_url IN (SELECT partial_url FROM root_page_partial_urls); \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/alter_open_pages_view.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/alter_open_pages_view.sql new file mode 100644 index 0000000000..ed34ebcb02 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/alter_open_pages_view.sql @@ -0,0 +1,16 @@ +DROP VIEW IF EXISTS open_pages; + +CREATE VIEW open_pages AS +SELECT DISTINCT ON (p.bucket_id) p.page_id, + p.bucket_id, + p.partial_url, + v.page_size, + count(pm.member_id) AS assigned_members +FROM pages p + JOIN buckets b ON b.bucket_id = p.bucket_id + JOIN views v ON v.view_id = b.view_id + LEFT JOIN page_members pm ON pm.page_id = p.page_id +WHERE NOT p.immutable +GROUP BY p.page_id, v.page_size, p.bucket_id, p.is_root +ORDER BY p.bucket_id, p.is_root +; diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/master.xml new file mode 100644 index 0000000000..67f0e9a8bb --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/master.xml @@ -0,0 +1,9 @@ + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml index a46826b4da..9d40834628 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml @@ -12,5 +12,6 @@ + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-maintenance-repository/pom.xml b/ldes-server-infra-postgres/postgres-maintenance-repository/pom.xml index 600c42df2e..20cd69c2e2 100644 --- a/ldes-server-infra-postgres/postgres-maintenance-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-maintenance-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.6.0 + 3.6.1-SNAPSHOT postgres-maintenance-repository diff --git a/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/entity/CompactionPageEntity.java b/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/entity/CompactionPageEntity.java index c85cf9539e..a6808b6249 100644 --- a/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/entity/CompactionPageEntity.java +++ b/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/entity/CompactionPageEntity.java @@ -27,6 +27,9 @@ public class CompactionPageEntity { @Column(name = "partial_url", nullable = false, unique = true) private String partialUrl; + @Column(name = "is_root", nullable = false, columnDefinition = "BOOLEAN") + private boolean isRoot; + public CompactionPageEntity() { } diff --git a/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/repository/CompactionPageEntityRepository.java b/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/repository/CompactionPageEntityRepository.java index b7a09057a5..581352782b 100644 --- a/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/repository/CompactionPageEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/repository/CompactionPageEntityRepository.java @@ -22,6 +22,7 @@ SELECT p.page_id as fragmentId, COUNT(*) AS size, r.to_page_id AS toPage, p.buck WHERE c.name = :collectionName AND v.name = :viewName AND p.expiration IS NULL AND p.immutable + AND NOT p.is_root GROUP BY p.page_id, r.to_page_id HAVING COUNT(*) < :capacityPerPage """, nativeQuery = true) diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml index 634051dec8..210bb3dad4 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.6.0 + 3.6.1-SNAPSHOT postgres-pagination-repository diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java index 7bd32d38c5..bc21bfb7ea 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java @@ -30,6 +30,9 @@ public class PageEntity { @Column(name = "partial_url", nullable = false, unique = true) private String partialUrl; + @Column(name = "is_root", nullable = false, columnDefinition = "BOOLEAN") + private boolean isRoot; + @OneToMany(mappedBy = "fromPage") private List relations; diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java index 02054ee99f..281d6e14d4 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java @@ -7,7 +7,7 @@ public interface PageEntityRepository extends JpaRepository { @Modifying - @Query(value = "UPDATE pages SET immutable = true WHERE page_id = ?", nativeQuery = true) + @Query(value = "UPDATE pages SET immutable = true WHERE page_id = ? AND NOT is_root", nativeQuery = true) void setPageImmutable(long pageId); @Modifying diff --git a/ldes-server-ingest/ldes-server-ingest-common/pom.xml b/ldes-server-ingest/ldes-server-ingest-common/pom.xml index e1ec45a3a6..1d7e4f9cdf 100644 --- a/ldes-server-ingest/ldes-server-ingest-common/pom.xml +++ b/ldes-server-ingest/ldes-server-ingest-common/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-ingest - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-ingest-common diff --git a/ldes-server-ingest/ldes-server-ingest-kafka/pom.xml b/ldes-server-ingest/ldes-server-ingest-kafka/pom.xml index 63a25a0224..94c3498751 100644 --- a/ldes-server-ingest/ldes-server-ingest-kafka/pom.xml +++ b/ldes-server-ingest/ldes-server-ingest-kafka/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-ingest - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-ingest-kafka diff --git a/ldes-server-ingest/ldes-server-ingest-rest/pom.xml b/ldes-server-ingest/ldes-server-ingest-rest/pom.xml index 6df099f2ab..0f60105660 100644 --- a/ldes-server-ingest/ldes-server-ingest-rest/pom.xml +++ b/ldes-server-ingest/ldes-server-ingest-rest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-ingest - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-ingest-rest diff --git a/ldes-server-ingest/pom.xml b/ldes-server-ingest/pom.xml index 6b337923dc..4b37762c8e 100644 --- a/ldes-server-ingest/pom.xml +++ b/ldes-server-ingest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.6.0 + 3.6.1-SNAPSHOT pom diff --git a/ldes-server-integration-test/pom.xml b/ldes-server-integration-test/pom.xml index e7ed013a38..4b2cd88271 100644 --- a/ldes-server-integration-test/pom.xml +++ b/ldes-server-integration-test/pom.xml @@ -6,14 +6,14 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.6.0 + 3.6.1-SNAPSHOT ldes-server-integration-test 1.18.3 - 3.6.0 + 3.7.1 1.18.3 diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java index d22bbdf796..323215a136 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java @@ -2,8 +2,15 @@ import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageRelationEntity; +import be.vlaanderen.informatievlaanderen.ldes.server.resultactionsextensions.ResponseToModelConverter; +import io.cucumber.java.After; +import io.cucumber.java.Before; import io.cucumber.java.en.And; import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import org.apache.jena.rdf.model.ResourceFactory; +import org.apache.jena.riot.Lang; +import org.apache.jena.vocabulary.RDF; import java.io.IOException; import java.net.URISyntaxException; @@ -17,17 +24,34 @@ import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; import java.util.stream.Collectors; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; import static org.awaitility.Awaitility.await; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @SuppressWarnings("java:S3415") public class CompactionServiceSteps extends LdesServerIntegrationTest { private int versionIncremeter = 1; + private ScheduledExecutorService executorService; + private Future seedingTask; + + @Before + public void setup() { + executorService = Executors.newSingleThreadScheduledExecutor(); + } + + @After + public void cleanup() { + executorService.shutdown(); + } @And("I ingest {int} members of different versions") public void ingestDifferentVersions(int amount) throws Exception { @@ -144,4 +168,70 @@ private boolean isValidUuid(String pageNumber) { return false; } } + + @And("I start seeding {int} members every {int} seconds") + public void iStartSeedingMembersEverySeconds(int numberOfMembers, int seconds) { + seedingTask = executorService.scheduleAtFixedRate(() -> ingestNumberOfVersionObjects(numberOfMembers), 0, seconds, SECONDS); + } + + private void ingestNumberOfVersionObjects(int numberOfMembers) { + try { + String memberTemplate = readMemberTemplate("data/input/members/observation.template.json"); + for (int i = 0; i < numberOfMembers; i++) { + String memberContent = memberTemplate + .replace("ID", String.valueOf(i)) + .replace("DATETIME", getCurrentTimestamp()); + mockMvc.perform(post("/observations") + .contentType(Lang.JSONLD.getHeaderString()) + .content(memberContent)) + .andExpect(status().is2xxSuccessful()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Then("I wait until {int} members are ingested") + public void iWaitUntilMembersAreIngested(int number) { + await().atMost(Duration.ofMinutes(3)) + .pollInterval(Duration.ofSeconds(15)) + .untilAsserted(() -> assertThat(jdbcTemplate.queryForObject("SELECT COUNT(*) FROM members", Long.class)).isEqualTo(number)); + } + + @Then("I stop seeding members") + public void iStopSeedingMembers() { + seedingTask.cancel(true); + } + + + @When("I wait until the first page does not exits anymore") + public void iWaitUntilThePageWithPageNumberDoesNotExitsAnymore() { + await() + .atMost(Duration.ofMinutes(2)) + .pollInterval(Duration.ofSeconds(5)) + .untilAsserted(() -> mockMvc.perform(get("/observations/time-based?pageNumber=1")) + .andExpect(status().isNotFound())); + } + + @And("I only have one open page") + public void iOnlyHaveOneOpenPage() { + final Long openPageCount = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM open_pages", Long.class); + assertThat(openPageCount).isEqualTo(1); + } + + @Then("the root page points to a compacted page") + public void theRootPagePointsToACompactedPage() throws Exception { + final var response = mockMvc.perform(get("/observations/time-based").accept(Lang.NQ.getHeaderString())) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse(); + final String pageNumber = new ResponseToModelConverter(response).convert() + .listSubjectsWithProperty(RDF.type, ResourceFactory.createProperty("https://w3id.org/tree#Relation")) + .nextResource() + .listProperties(ResourceFactory.createProperty("https://w3id.org/tree#node")) + .nextStatement() + .getResource() + .getLocalName(); + assertThatNoException().isThrownBy(() -> UUID.fromString(pageNumber)); + } } diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java index 0ec36d412f..e94381fcda 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java @@ -15,10 +15,13 @@ import org.apache.jena.rdf.model.Resource; import org.apache.jena.riot.*; import org.apache.jena.vocabulary.RDF; +import org.assertj.core.api.InstanceOfAssertFactories; import org.awaitility.Awaitility; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobInstance; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.launch.NoSuchJobException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.mock.web.MockHttpServletResponse; @@ -262,7 +265,7 @@ public void firstFragmentOfViewContainsMembers(String view, String collection, l Model fragmentPage = fetchFragment(fragmentUrl); return fragmentPage.listObjectsOfProperty(createProperty("https://w3id.org/tree#member")) - .toList().size() == expectedMemberCount; + .toList().size() == expectedMemberCount; }); } @@ -270,9 +273,9 @@ public void firstFragmentOfViewContainsMembers(String view, String collection, l public void theLDESContainsMembers(String collection, int expectedMemberCount) { await().atMost(60, SECONDS) .until(() -> jdbcTemplate - .queryForObject("SELECT COUNT(*) FROM members m JOIN collections c ON m.collection_id = c.collection_id WHERE c.name = ?", - Integer.class, collection) - == expectedMemberCount); + .queryForObject("SELECT COUNT(*) FROM members m JOIN collections c ON m.collection_id = c.collection_id WHERE c.name = ?", + Integer.class, collection) + == expectedMemberCount); } @After @@ -367,10 +370,27 @@ public void theResponseFromRequestingTheUrlDoesContainAJsonFile(String message, @And("the background processes did not fail") public void theBackgroundProcessesDidNotFail() { - JobInstance lastJobInstance = Objects.requireNonNull(jobExplorer.getLastJobInstance(MAINTENANCE_JOB)); - JobExecution lastJobExecution = Objects.requireNonNull(jobExplorer.getLastJobExecution(lastJobInstance)); - boolean hasFailedExecutions = lastJobExecution.getStepExecutions().stream() - .anyMatch(stepExecution -> stepExecution.getStatus().equals(BatchStatus.FAILED)); - assertThat(hasFailedExecutions).isFalse(); + assertThat(jobExplorer.getLastJobInstance(MAINTENANCE_JOB)) + .isNotNull() + .extracting(jobExplorer::getLastJobExecution) + .isNotNull() + .extracting(JobExecution::getStepExecutions, InstanceOfAssertFactories.list(StepExecution.class)) + .map(StepExecution::getStatus) + .doesNotContain(BatchStatus.FAILED); + } + + @And("the batch tables has been cleaned") + public void theBatchTablesHasBeenCleaned() throws NoSuchJobException { + final JobInstance lastJobInstance = jobExplorer.getLastJobInstance(MAINTENANCE_JOB); + assertThat(lastJobInstance).isNotNull(); + await() + .untilAsserted(() -> assertThat(jobExplorer.getLastJobExecution(lastJobInstance)) + .isNotNull() + .extracting(JobExecution::isRunning, InstanceOfAssertFactories.BOOLEAN) + .isFalse()); + assertThat(jobExplorer.getJobInstanceCount(MAINTENANCE_JOB)).isEqualTo(1); + assertThat(jobExplorer.getJobNames()) + .doesNotContain("fragmentation") + .hasSize(1); } } diff --git a/ldes-server-integration-test/src/test/resources/application-postgres-test.yml b/ldes-server-integration-test/src/test/resources/application-postgres-test.yml index cc3f2571b6..8ebf6316cf 100644 --- a/ldes-server-integration-test/src/test/resources/application-postgres-test.yml +++ b/ldes-server-integration-test/src/test/resources/application-postgres-test.yml @@ -1,8 +1,8 @@ ldes-server: host-name: "http://localhost:8080" - compaction-duration: "*/10 * * * * *" fragmentation-cron: "*/10 * * * * *" - maintenance-cron: "*/10 * * * * *" + maintenance-cron: "*/5 * * * * *" + compaction-duration: PT2M springdoc.swaggerui.path: "/swagger" management: diff --git a/ldes-server-integration-test/src/test/resources/data/input/eventstreams/compaction/observations.ttl b/ldes-server-integration-test/src/test/resources/data/input/eventstreams/compaction/observations.ttl new file mode 100644 index 0000000000..468618430c --- /dev/null +++ b/ldes-server-integration-test/src/test/resources/data/input/eventstreams/compaction/observations.ttl @@ -0,0 +1,31 @@ +@prefix ldes: . +@prefix tree: . +@prefix sh: . +@prefix ex: . +@prefix xsd: . +@prefix dcterms: . + + a ldes:EventStream ; + ldes:timestampPath dcterms:created ; + ldes:versionOfPath dcterms:isVersionOf ; + tree:shape [ a sh:NodeShape ] ; + tree:view ; + ldes:eventSource [ + a ldes:EventSource ; + ldes:retentionPolicy [ + a ldes:DurationAgoPolicy ; + tree:value "PT2M"^^xsd:duration + ] ; + ] . + + a tree:Node ; + tree:viewDescription [ + a tree:ViewDescription ; + tree:fragmentationStrategy () ; + tree:pageSize "7"^^xsd:integer ; + ldes:retentionPolicy [ + a ldes:DurationAgoPolicy ; + tree:value "PT90S"^^xsd:duration + ] + ] +. \ No newline at end of file diff --git a/ldes-server-integration-test/src/test/resources/data/input/members/observation.template.json b/ldes-server-integration-test/src/test/resources/data/input/members/observation.template.json new file mode 100644 index 0000000000..799083b4f4 --- /dev/null +++ b/ldes-server-integration-test/src/test/resources/data/input/members/observation.template.json @@ -0,0 +1,20 @@ +{ + "@context": { + "@base": "http://example.org/id/", + "@vocab": "http://purl.org/dc/terms/", + "isVersionOf": { + "@type": "@id" + } + }, + "@id": "ID#DATETIME", + "@type": [ + "something" + ], + "created": [ + { + "@value": "DATETIME", + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + ], + "isVersionOf": "http://example.org/id/ID" +} \ No newline at end of file diff --git a/ldes-server-integration-test/src/test/resources/features/maintenance/compaction.feature b/ldes-server-integration-test/src/test/resources/features/maintenance/compaction.feature index 959f094468..c067384d59 100644 --- a/ldes-server-integration-test/src/test/resources/features/maintenance/compaction.feature +++ b/ldes-server-integration-test/src/test/resources/features/maintenance/compaction.feature @@ -1,13 +1,11 @@ Feature: LDES Server Compaction - Background: + Scenario: Execution Compaction Given I create the eventstream "data/input/eventstreams/compaction/mobility-hindrances_paginated_5.ttl" And I ingest 6 members of different versions And I ingest 5 members of the same version And I ingest 5 members of the same version And I ingest 3 members of different versions - - Scenario: Execution Compaction Then I wait until all members are fragmented Then wait until no fragments can be compacted And verify there are 5 pages @@ -21,4 +19,14 @@ Feature: LDES Server Compaction And verify the following pages no longer exist | 2 | | 3 | - And the background processes did not fail \ No newline at end of file + And the background processes did not fail + + Scenario: Retention Compaction And Deletion works fine together + Given I create the eventstream "data/input/eventstreams/compaction/observations.ttl" + And I start seeding 5 members every 15 seconds + When I wait until the first page does not exits anymore + Then the root page points to a compacted page + And I only have one open page + Then I stop seeding members + And I delete the eventstream "observations" + And the background processes did not fail diff --git a/ldes-server-integration-test/src/test/resources/features/maintenance/retention.feature b/ldes-server-integration-test/src/test/resources/features/maintenance/retention.feature index 304468c3f3..2ce9f8e17f 100644 --- a/ldes-server-integration-test/src/test/resources/features/maintenance/retention.feature +++ b/ldes-server-integration-test/src/test/resources/features/maintenance/retention.feature @@ -8,6 +8,7 @@ Feature: LDES Server Retention # Since all added members' timestamp values equal to their ingestion date, they should be removed after 15 seconds Then the first fragment of the "paged" view in collection contains 0 members And the background processes did not fail + And the batch tables has been cleaned Examples: | eventStreamDescriptionFile | template | collection | ingestedMemberCount | | "data/input/eventstreams/retention/mobility-hindrances_timebased.ttl" | "data/input/members/mob-hind.template.ttl" | "mobility-hindrances" | 30 | @@ -19,6 +20,7 @@ Feature: LDES Server Retention When I ingest 30 members of template