From 32e3b6a3ca8ace4fc1abe12b1f3b4e3bd83c30fb Mon Sep 17 00:00:00 2001 From: Marina Gourtovaia Date: Wed, 11 Sep 2024 15:07:05 +0100 Subject: [PATCH 1/9] Updated the expected checksum for perbrew installer --- .github/workflows/perlbrew.sha256 | 2 +- Changes | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/perlbrew.sha256 b/.github/workflows/perlbrew.sha256 index d9992912..2623e110 100644 --- a/.github/workflows/perlbrew.sha256 +++ b/.github/workflows/perlbrew.sha256 @@ -1 +1 @@ -c3996e4fae37a0ae01839cdd73752fb7b17e81bac2a8b39712463a7d518c4945 perlbrew.sh +8f254651d2eee188199b3355228eb67166974716081b794ca93b69c8f949c38d perlbrew.sh diff --git a/Changes b/Changes index a6e5b955..5ec4bf7a 100644 --- a/Changes +++ b/Changes @@ -1,5 +1,13 @@ LIST OF CHANGES FOR NPG-QC PACKAGE + - Following a release on 07/09/2024, see https://metacpan.org/dist/App-perlbrew/changes + the checksum of the script served by https://install.perlbrew.pl had changed. + https://install.perlbrew.pl is a redirect to raw + https://github.com/gugod/App-perlbrew/blob/master/perlbrew-install, so + the change originates from GitHub and can be trusted. Our CI flow compares + the checksum of the downloaded script to the expected value. We now store + an updated expeced checksum value, which corresponds to the latest release. + release 72.2.0 (2024-08-30) - npg_qc::autoqc::check::review: 1. To enable access to information about a sequencing run (from RunInfo.xml, From 65cf0df357ab819a3c3c1df00adeca461fead9ab Mon Sep 17 00:00:00 2001 From: Marina Gourtovaia Date: Thu, 12 Sep 2024 11:57:52 +0100 Subject: [PATCH 2/9] Updated deprecated v2 GitHub action to v3 --- .github/workflows/run-tests.yml | 2 +- Changes | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 59f2b2b1..2f919c4a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -123,7 +123,7 @@ jobs: - name: "Archive CPAN logs on failure" if: ${{ failure() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: cpan_log path: ~/.cpanm/work/*/build.log diff --git a/Changes b/Changes index 5ec4bf7a..9e2a5ee3 100644 --- a/Changes +++ b/Changes @@ -1,5 +1,6 @@ LIST OF CHANGES FOR NPG-QC PACKAGE + - GitHub CI - updated deprecated v2 runner action to v3 - Following a release on 07/09/2024, see https://metacpan.org/dist/App-perlbrew/changes the checksum of the script served by https://install.perlbrew.pl had changed. https://install.perlbrew.pl is a redirect to raw From 3ed53f21e667343c6188923289ef693074ab5b1c Mon Sep 17 00:00:00 2001 From: Marina Gourtovaia Date: Wed, 11 Sep 2024 10:11:38 +0100 Subject: [PATCH 3/9] Made library_type optional in the review check. --- Changes | 7 ++++- lib/npg_qc/Schema/Result/Review.pm | 14 ++++----- lib/npg_qc/autoqc/checks/review.pm | 8 ++--- t/50-schema-result-Review.t | 49 ++++++++++++------------------ t/60-autoqc-checks-review.t | 16 +++++++++- 5 files changed, 51 insertions(+), 43 deletions(-) diff --git a/Changes b/Changes index 9e2a5ee3..3c7beb8c 100644 --- a/Changes +++ b/Changes @@ -7,7 +7,12 @@ LIST OF CHANGES FOR NPG-QC PACKAGE https://github.com/gugod/App-perlbrew/blob/master/perlbrew-install, so the change originates from GitHub and can be trusted. Our CI flow compares the checksum of the downloaded script to the expected value. We now store - an updated expeced checksum value, which corresponds to the latest release. + an updated expected checksum value, which corresponds to the latest release. + - npg_qc::autoqc::check::review: + The forthcoming lane-level evaluations are impossible if the code continues + to error when library_type attribute is not defined for the entity under + consideration. The library_type attribute is now set when possible, no error + if it is undefined. release 72.2.0 (2024-08-30) - npg_qc::autoqc::check::review: diff --git a/lib/npg_qc/Schema/Result/Review.pm b/lib/npg_qc/Schema/Result/Review.pm index 1aac0e56..7ce04be9 100644 --- a/lib/npg_qc/Schema/Result/Review.pm +++ b/lib/npg_qc/Schema/Result/Review.pm @@ -357,14 +357,12 @@ around [qw/update insert/] => sub { ##### # Do not accept half-baked results, ie if we have evaluation - # results, we should also have library type and criteria. + # results, we should also have criteria. if ($data->{'evaluation_results'} and keys %{$data->{'evaluation_results'}}) { - foreach my $name (qw/library_type criteria/) { - my $value = $data->{$name}; - my $m = "Evaluation results present, but $name absent"; - $value or croak $m; - ((not ref $value) or keys %{$value}) or croak $m; - } + my $value = $data->{'criteria'}; + my $m = 'Evaluation results present, but criteria absent'; + $value or croak $m; + ((not ref $value) or keys %{$value}) or croak $m; } ##### @@ -475,7 +473,7 @@ Marina Gourtovaia Emg8@sanger.ac.ukE =head1 LICENSE AND COPYRIGHT -Copyright (C) 2019,2020 Genome Research Ltd. +Copyright (C) 2019,2020, 2924 Genome Research Ltd. This file is part of NPG. diff --git a/lib/npg_qc/autoqc/checks/review.pm b/lib/npg_qc/autoqc/checks/review.pm index 08a24267..4c1972ab 100644 --- a/lib/npg_qc/autoqc/checks/review.pm +++ b/lib/npg_qc/autoqc/checks/review.pm @@ -568,11 +568,11 @@ has '_criteria' => ( sub _build__criteria { my $self = shift; - # Save redundant library_type. - # TODO: Save details about applicability instead. + # Library type might be undefined. Example - lane level object. my $lib_type = $self->lims->library_type; - $lib_type or croak 'Library type is not defined for ' . $self->_entity_desc; - $self->result->library_type($lib_type); + if ($lib_type) { + $self->result->library_type($lib_type); + } my $num_criteria = scalar @{$self->_applicable_criteria}; if ($num_criteria == 0) { diff --git a/t/50-schema-result-Review.t b/t/50-schema-result-Review.t index 6be7101c..789bedca 100644 --- a/t/50-schema-result-Review.t +++ b/t/50-schema-result-Review.t @@ -19,19 +19,9 @@ my $schema = Moose::Meta::Class->create_anon_class( ->new_object({})->create_test_db(q[npg_qc::Schema], 't/data/fixtures'); subtest 'reject incomplete results on insert' => sub { - plan tests => 4; + plan tests => 2; my $values = { - evaluation_results => {"e1"=>1,"e2"=>0}, - criteria => {"and"=>["e1","e2"]}, - pass => 0, - path => 't/data' - }; - throws_ok {$schema->resultset($table)->create($values)} - qr/Evaluation results present, but library_type absent/, - 'library type is not defined - error on insert'; - - $values = { library_type => 'common_type', criteria => {}, evaluation_results => {"e1"=>1,"e2"=>0}, @@ -41,18 +31,6 @@ subtest 'reject incomplete results on insert' => sub { qr/Evaluation results present, but criteria absent/, 'criteria are not defined - error on insert'; - $values = { - evaluation_results => {"e1"=>1,"e2"=>0}, - criteria => {}, - qc_outcome => - {"mqc_outcome"=>"Rejected final","timestamp"=>"2018-06-03T12:53:46+0000","username"=>"robo_qc"}, - pass => 0, - path => 't/data' - }; - throws_ok {$schema->resultset($table)->create($values)} - qr/Evaluation results present, but library_type absent/, - 'criteria and library type are not defined - error on insert'; - $values = { library_type => 'common_type', evaluation_results => {"e1"=>1,"e2"=>0}, @@ -68,13 +46,27 @@ subtest 'reject incomplete results on insert' => sub { }; subtest 'insert a basic record, do not allow incomplete data in update' => sub { - plan tests => 9; + plan tests => 10; my $id_seq_composition = t::autoqc_util::find_or_save_composition( $schema, {'id_run' => 1111, 'position' => 1, - 'tag_index' => 8}); + 'tag_index' => 7}); my $values = { + evaluation_results => {"e1"=>1,"e2"=>0}, + criteria => {"and"=>["e1","e2"]}, + pass => 0, + path => 't/data', + id_seq_composition => $id_seq_composition + }; + lives_ok {$schema->resultset($table)->create($values)} + 'library type is not defined - no error on insert'; + + $id_seq_composition = t::autoqc_util::find_or_save_composition( + $schema, {'id_run' => 1111, + 'position' => 1, + 'tag_index' => 8}); + $values = { evaluation_results => {}, criteria => {}, qc_outcome => {}, @@ -108,9 +100,8 @@ subtest 'insert a basic record, do not allow incomplete data in update' => sub { pass => 0, path => 't/data' }; - throws_ok {$new->update($values)} - qr/Evaluation results present, but library_type absent/, - 'library type is not defined - error on update'; + lives_ok {$new->update($values)} + 'library type is not defined - no error on update'; $values = { id_seq_composition => $id_seq_composition, @@ -277,7 +268,7 @@ subtest 'a full insert/update record with mqc outcome' => sub { }; my $created=$schema->resultset($table)->create($values); $rs = $schema->resultset($table)->search({}); - is ($rs->count, 2, q[two rows in the table]); + is ($rs->count, 3, q[three rows in the table]); $mqc_rs = $schema->resultset($mqc_table)->search({id_seq_composition => $id_seq_composition}); is ($mqc_rs->count, 1, 'one mqc record for the entity'); diff --git a/t/60-autoqc-checks-review.t b/t/60-autoqc-checks-review.t index 36b769a3..b2f74853 100644 --- a/t/60-autoqc-checks-review.t +++ b/t/60-autoqc-checks-review.t @@ -401,7 +401,7 @@ subtest 'single expression evaluation' => sub { }; subtest 'evaluation within the execute method' => sub { - plan tests => 42; + plan tests => 48; local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = 't/data/autoqc/review/samplesheet_29524.csv'; @@ -439,6 +439,8 @@ subtest 'evaluation within the execute method' => sub { foreach my $check (@check_objects) { lives_ok { $check->execute } 'execute method runs OK'; is ($check->result->pass, 1, 'result pass attribute is set to 1'); + is ($check->result->library_type, 'HiSeqX PCR free', + 'result library_type attribute is set correctly'); my %expected = map { $_ => 1 } @{$criteria_list}; is_deeply ($check->result->evaluation_results(), \%expected, 'evaluation results are saved'); @@ -454,7 +456,19 @@ subtest 'evaluation within the execute method' => sub { $count++; } + # Undefined library type should not be a problem. + my $lane_lims = (st::api::lims->new(id_run=> 29524)->children())[0]; + is ($lane_lims->library_type, undef, 'library type is undefined on lane level'); my $check = npg_qc::autoqc::checks::review->new( + conf_path => $test_data_dir, + qc_in => $test_data_dir, + rpt_list => $rpt_list, + lims => $lane_lims + ); + lives_ok { $check->execute } 'execute method runs OK'; + is ($check->result->library_type, undef, 'library_type attribute is unset'); + + $check = npg_qc::autoqc::checks::review->new( conf_path => "$test_data_dir/unknown_qc_type", qc_in => $dir, rpt_list => $rpt_list); From 2a3fcb9be4ac19541425c945b01153cff93c4465 Mon Sep 17 00:00:00 2001 From: Avnish Pratap Singh Date: Thu, 12 Sep 2024 12:22:14 +0100 Subject: [PATCH 4/9] Automatic confluence update --- .gitlab-ci.yml | 33 +++++++++++++++++++++++++++++++ Changes | 2 ++ docs/CtoA_substitution_metrics.md | 17 ++++++++++++++++ docs/includes/repo-metadata.md | 1 + docs/qc_outcomes_change_howto.md | 17 ++++++++++++++++ 5 files changed, 70 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 docs/includes/repo-metadata.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..c01c624a --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,33 @@ +stages: + - generate + - publish + +generate-job: + stage: generate + tags: + - autoscale + rules: + - if: $CI_COMMIT_BRANCH == "devel" || $CI_COMMIT_BRANCH == "master" + changes: + - docs/**/* + variables: + REPO_METADATA_FILE: docs/includes/repo-metadata.md + script: > + echo "Created from: [$CI_PROJECT_NAMESPACE / $CI_PROJECT_TITLE]($CI_PROJECT_URL) Version: $(git describe --always)" > "$REPO_METADATA_FILE" + artifacts: + paths: + - "$REPO_METADATA_FILE" + +mark-job: + stage: publish + rules: + - if: $CI_COMMIT_BRANCH == "devel" || $CI_COMMIT_BRANCH == "master" + changes: + - docs/**/* + image: wsinpg/mark:latest + tags: + - autoscale + script: > + for f in $(find . -name includes -type d -prune -o -name 'README*.md' -prune -o -name '*.md' -print0 | xargs -0); do + echo 'username = ""' && echo "password = '$CONFLUENCE_PERSONAL_ACCESS_TOKEN'" | mark -c /dev/stdin -b https://ssg-confluence.internal.sanger.ac.uk -f "$f" --trace; + done diff --git a/Changes b/Changes index 9e2a5ee3..d11510e4 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ LIST OF CHANGES FOR NPG-QC PACKAGE - GitHub CI - updated deprecated v2 runner action to v3 + - docs + - automated creation/update of confluence pages for docs - Following a release on 07/09/2024, see https://metacpan.org/dist/App-perlbrew/changes the checksum of the script served by https://install.perlbrew.pl had changed. https://install.perlbrew.pl is a redirect to raw diff --git a/docs/CtoA_substitution_metrics.md b/docs/CtoA_substitution_metrics.md index e9f0ede0..d952b6f4 100644 --- a/docs/CtoA_substitution_metrics.md +++ b/docs/CtoA_substitution_metrics.md @@ -1,3 +1,20 @@ + + + + + + +:box:info:Note:This page is automatically generated; any edits will be overwritten: + +###### Repository information + + + # C->A short read sequencing substitution metrics - Author - Irina Abnizova diff --git a/docs/includes/repo-metadata.md b/docs/includes/repo-metadata.md new file mode 100644 index 00000000..e8cc8a76 --- /dev/null +++ b/docs/includes/repo-metadata.md @@ -0,0 +1 @@ +Created from: - Version: - \ No newline at end of file diff --git a/docs/qc_outcomes_change_howto.md b/docs/qc_outcomes_change_howto.md index 8c474dda..d1e732a1 100644 --- a/docs/qc_outcomes_change_howto.md +++ b/docs/qc_outcomes_change_howto.md @@ -1,3 +1,20 @@ + + + + + + +:box:info:Note:This page is automatically generated; any edits will be overwritten: + +###### Repository information + + + # Guidance for Changing QC Outcomes - Use the correct RT ticket number. From 0c371e81606f5e5e22231a13b79a6fc3167fb9be Mon Sep 17 00:00:00 2001 From: Avnish Pratap Singh Date: Tue, 1 Oct 2024 09:31:43 +0100 Subject: [PATCH 5/9] Added dependabot.yml to auto-update GitHub actions --- .github/dependabot.yml | 10 ++++++++++ Changes | 1 + 2 files changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..619e9bd4 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +# Referenced from: +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot + +version: 2 +updates: + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/Changes b/Changes index f03a4be0..c6e4aa27 100644 --- a/Changes +++ b/Changes @@ -1,5 +1,6 @@ LIST OF CHANGES FOR NPG-QC PACKAGE + - Added .github/dependabot.yml file to auto-update GitHub actions - GitHub CI - updated deprecated v2 runner action to v3 - docs - automated creation/update of confluence pages for docs From 9b5b341a60295c4becde8eece0f064010a8f3dc2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:31:32 +0000 Subject: [PATCH 6/9] Bump actions/upload-artifact from 3 to 4 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3 to 4. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 2f919c4a..cf27952f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -123,7 +123,7 @@ jobs: - name: "Archive CPAN logs on failure" if: ${{ failure() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: cpan_log path: ~/.cpanm/work/*/build.log From 030ac81e0d220b2cfb1b3bb143e7f2bb2b2db44a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:31:34 +0000 Subject: [PATCH 7/9] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 2f919c4a..6fd47abe 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -26,7 +26,7 @@ jobs: node: ['14.8.0'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: "Install OS dependencies" run: | From 009746b2e670ddc5b3d761820f72d1c64ca48b43 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 11:25:04 +0000 Subject: [PATCH 8/9] Bump actions/cache from 3 to 4 Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 6fd47abe..d648e1dc 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -41,7 +41,7 @@ jobs: - name: "Cache Conda" id: minicondaCache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/conda/pkgs @@ -67,7 +67,7 @@ jobs: - name: "Cache Perl" id: cache-perl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.PERL_CACHE }} key: ${{ runner.os }}-${{ matrix.perl }}-perl From 18c5b62a2eedf2b6c8e62859b9ec80aa76986177 Mon Sep 17 00:00:00 2001 From: jmtcsngr Date: Fri, 4 Oct 2024 13:51:22 +0100 Subject: [PATCH 9/9] prep release 72.2.1 --- Changes | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Changes b/Changes index c6e4aa27..eff1f0ea 100644 --- a/Changes +++ b/Changes @@ -1,9 +1,10 @@ LIST OF CHANGES FOR NPG-QC PACKAGE +release 72.2.1 (2024-10-04) - Added .github/dependabot.yml file to auto-update GitHub actions - GitHub CI - updated deprecated v2 runner action to v3 - docs - - automated creation/update of confluence pages for docs + - automated creation/update of confluence pages for docs - Following a release on 07/09/2024, see https://metacpan.org/dist/App-perlbrew/changes the checksum of the script served by https://install.perlbrew.pl had changed. https://install.perlbrew.pl is a redirect to raw @@ -12,10 +13,10 @@ LIST OF CHANGES FOR NPG-QC PACKAGE the checksum of the downloaded script to the expected value. We now store an updated expected checksum value, which corresponds to the latest release. - npg_qc::autoqc::check::review: - The forthcoming lane-level evaluations are impossible if the code continues - to error when library_type attribute is not defined for the entity under - consideration. The library_type attribute is now set when possible, no error - if it is undefined. + The forthcoming lane-level evaluations are impossible if the code continues + to error when library_type attribute is not defined for the entity under + consideration. The library_type attribute is now set when possible, no error + if it is undefined. release 72.2.0 (2024-08-30) - npg_qc::autoqc::check::review: