From 6f7f0730e7a279613567f7c05454a8f088b3db43 Mon Sep 17 00:00:00 2001 From: Konstantinos <> Date: Mon, 8 Jul 2019 05:21:31 +0200 Subject: [PATCH 1/9] Restructure codebase, add/modify distribution-related support files, write simple invocation unittest for 'create-album' cli app and remove unused dataset txt files from 'music_album_creation/format_classification' dir Update the 'update_url' parameter with the 'v1.0.4.tar.gz' release download url in the setup.py Update the 'download_url' parameter with the 'v1.0.6.tar.gz' release download-url in the setup.py Update the 'download_url' parameter with the 'v1.0.7.tar.gz' release download-url in the setup.py Add setup.cfg file pointing to the LICENSE and README files Add the 'bdist_wheel.universal' = 1 settings in the setup.cfg, to make a potential wheel universal Increment to 1.0.7a Write and include information in the CHANGELOG.rst file Restructure adding 'src' folder' Change invocation definiton of 'create_album.py' from '\#\!/usr/bin/python' to '\#\!/usr/bin/env python3' Expose certain objects from 'music_album_creation' package through __init__.py Add test for launcing the create-album cli program Put 'testing/test_create_album_program.py' into 'tests' directory WIP Increment to 1.0.8a, use name='music_album_creation' and remove 'download_url' parameter in setup.py Fix MANIFEST to compy with the addition of 'src' directory Remove the 'bdist_wheel.universal = 1' setting from setup.cfg Point to 'src' directory in setup.py and revert back to using the 'console_scripts' field of the 'entry_points' parameter Fix path string in 'test_classifier.py' Chnage string in setup.py WIP WIP Delete dataset files from 'format_classification' dir/package Move 'create_album' and change imports to relative Add 'from __future__ import absolute_import' statement in setup.py, fix 'entry_points' parameter in setup.py, add version in 'music_album_creation' package's __init__ file, fix cli.runner unittest for 'crate-album' console script Remove 'from __future__ import absolute_import' statement in setup.py and pass all unittests in python3.5 local environemnt Add create-album launching test and pass it Greatly enrich and fancify README Include AUTHORS.rst, everything under 'src' dir, everything under 'tests' dir and .travis.yml when creating a source distribution (sdist) through 'include' and 'graft' statements in MANIFEST.in file --- CHANGELOG.rst | 20 + MANIFEST.in | 17 +- README.rst | 95 +++ .../format_classification/dev-split.txt | 78 --- .../format_classification/test-split.txt | 78 --- .../format_classification/train-split.txt | 608 ------------------ setup.cfg | 3 + setup.py | 15 +- src/music_album_creation/__init__.py | 8 + .../album_segmentation.py | 0 .../music_album_creation}/create_album.py | 20 +- .../music_album_creation}/dialogs.py | 0 .../music_album_creation}/display-logo.sh | 0 .../music_album_creation}/downloading.py | 0 .../format_classification/__init__.py | 0 .../format_classification/data/dev-split.txt | 0 .../format_classification/data/model.pickle | Bin .../format_classification/data/model1 | Bin .../format_classification/data/test-split.txt | 0 .../data/train-split.txt | 0 .../format_classification/dataset.py | 0 .../tracks_format_classifier.py | 0 .../music_album_creation}/metadata.py | 0 .../music_album_creation}/tracks_parsing.py | 0 testing/__init__.py | 0 {music_album_creation => tests}/__init__.py | 0 {testing => tests}/know_your_enemy.mp3 | Bin {testing => tests}/test_classifier.py | 4 +- tests/test_create_album_program.py | 19 + {testing => tests}/test_downloading.py | 1 + {testing => tests}/test_segmenting.py | 1 + {testing => tests}/test_splitters.py | 2 +- 32 files changed, 184 insertions(+), 785 deletions(-) create mode 100644 CHANGELOG.rst delete mode 100644 music_album_creation/format_classification/dev-split.txt delete mode 100644 music_album_creation/format_classification/test-split.txt delete mode 100644 music_album_creation/format_classification/train-split.txt create mode 100644 setup.cfg create mode 100644 src/music_album_creation/__init__.py rename {music_album_creation => src/music_album_creation}/album_segmentation.py (100%) rename {music_album_creation => src/music_album_creation}/create_album.py (88%) rename {music_album_creation => src/music_album_creation}/dialogs.py (100%) rename {music_album_creation => src/music_album_creation}/display-logo.sh (100%) rename {music_album_creation => src/music_album_creation}/downloading.py (100%) rename {music_album_creation => src/music_album_creation}/format_classification/__init__.py (100%) rename {music_album_creation => src/music_album_creation}/format_classification/data/dev-split.txt (100%) rename {music_album_creation => src/music_album_creation}/format_classification/data/model.pickle (100%) rename {music_album_creation => src/music_album_creation}/format_classification/data/model1 (100%) rename {music_album_creation => src/music_album_creation}/format_classification/data/test-split.txt (100%) rename {music_album_creation => src/music_album_creation}/format_classification/data/train-split.txt (100%) rename {music_album_creation => src/music_album_creation}/format_classification/dataset.py (100%) rename {music_album_creation => src/music_album_creation}/format_classification/tracks_format_classifier.py (100%) rename {music_album_creation => src/music_album_creation}/metadata.py (100%) rename {music_album_creation => src/music_album_creation}/tracks_parsing.py (100%) delete mode 100644 testing/__init__.py rename {music_album_creation => tests}/__init__.py (100%) rename {testing => tests}/know_your_enemy.mp3 (100%) rename {testing => tests}/test_classifier.py (81%) create mode 100644 tests/test_create_album_program.py rename {testing => tests}/test_downloading.py (97%) rename {testing => tests}/test_segmenting.py (99%) rename {testing => tests}/test_splitters.py (97%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..7d88294 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,20 @@ +Changelog +========= + + +1.0.7a (2019-07-08) +------------------- + +Changes +^^^^^^^ + +- Add a universal wheel + + +1.0.7 (2019-07-08) +------------------- + +Changes: +^^^^^^^^ + +Initial release. \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 5fa630b..0ceabd4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,16 @@ include README.rst +include AUTHORS.rst +include CHANGELOG.rst + include LICENSE.txt -include music_album_creation/format_classification/data/* -include music_album_creation/display-logo.sh -include testing/know_your_enemy.mp3 \ No newline at end of file + +include src/music_album_creation/format_classification/data/* +include src/music_album_creation/display-logo.sh +include tests/know_your_enemy.mp3 + +graft src +graft tests + +include .travis.yml + +global-exclude *.py[cod] __pycache__ *.so *.dylib diff --git a/README.rst b/README.rst index 7713f7d..86b5006 100644 --- a/README.rst +++ b/README.rst @@ -2,3 +2,98 @@ Music Album Creator - CLI Application ===================================== Music Album Creator is a cli application aiming to automate the process of building an offline music library. + + +======== +Overview +======== + +.. start-badges + +.. list-table:: + :stub-columns: 1 + + * - docs + - |docs| + * - tests + - | |travis| + | |coveralls| + * - package + - | |version| |wheel| |supported-versions| |supported-implementations| + | |commits-since| +.. |docs| image:: https://readthedocs.org/projects/music-album-creator/badge/?style=flat + :target: https://readthedocs.org/projects/music-album-creator + :alt: Documentation Status + +.. |travis| image:: https://travis-ci.org/boromir674/music-album-creator.svg?branch=master + :alt: Travis-CI Build Status + :target: https://travis-ci.org/boromir674/music-album-creator + +.. |coveralls| image:: https://coveralls.io/repos/boromir674/music-album-creator/badge.svg?branch=master&service=github + :alt: Coverage Status + :target: https://coveralls.io/r/boromir674/music-album-creator + +.. |version| image:: https://img.shields.io/pypi/v/music-album-creator.svg + :alt: PyPI Package latest release + :target: https://pypi.org/project/music-album-creator + +.. |commits-since| image:: https://img.shields.io/github/commits-since/boromir674/music-album-creator/v0.svg + :alt: Commits since latest release + :target: https://github.com/boromir674/music-album-creator/compare/v0...master + +.. |wheel| image:: https://img.shields.io/pypi/wheel/music-album-creator.svg + :alt: PyPI Wheel + :target: https://pypi.org/project/music-album-creator + +.. |supported-versions| image:: https://img.shields.io/pypi/pyversions/music-album-creator.svg + :alt: Supported versions + :target: https://pypi.org/project/music-album-creator + +.. |supported-implementations| image:: https://img.shields.io/pypi/implementation/music-album-creator.svg + :alt: Supported implementations + :target: https://pypi.org/project/music-album-creator + + +.. end-badges + +A CLI application intending to automate offline music library building. + +* Free software: Apache Software License 2.0 + +Installation +============ + +:: + + pip install music-album-creator + +Documentation +============= + + +https://music-album-creator.readthedocs.io/ + + +Development +=========== + +To run the all tests run:: + + tox + +Note, to combine the coverage data from all the tox environments run: + +.. list-table:: + :widths: 10 90 + :stub-columns: 1 + + - - Windows + - :: + + set PYTEST_ADDOPTS=--cov-append + tox + + - - Other + - :: + + PYTEST_ADDOPTS=--cov-append tox diff --git a/music_album_creation/format_classification/dev-split.txt b/music_album_creation/format_classification/dev-split.txt deleted file mode 100644 index f6615b0..0000000 --- a/music_album_creation/format_classification/dev-split.txt +++ /dev/null @@ -1,78 +0,0 @@ -11 2479.621224484 1 -11 12200 0 -13 3835.324081627 1 -13 22598 0 -9 2870.04 1 -9 11120 0 -9 3007.0595918329996 1 -9 13044 0 -11 2773.1000000000004 1 -11 12357 0 -11 2857.9787755039997 1 -11 13446 0 -6 1672.2285714260001 1 -6 3760 0 -5 1888.5 1 -5 5440 0 -13 2063.96081632 1 -13 12193 0 -8 2900.2 1 -8 10668 0 -11 2793.0000000000005 1 -11 13518 0 -9 2523.6000000000004 1 -9 7728 0 -8 2177.2000000000003 1 -8 7588 0 -12 4084.5 1 -12 21180 0 -20 4309.900000000001 1 -20 40519 0 -12 3335.242448978999 1 -12 18094 0 -10 2177.436734688 1 -10 10501 0 -3 869.2 1 -3 1073 0 -4 2417.3999999999996 1 -4 3115 0 -16 4332.068571422 1 -16 30207 0 -8 2574.96 1 -8 9460 0 -13 3648.8999999999996 1 -13 21140 0 -9 2671.542857138 1 -9 9961 0 -14 3598.1000000000004 1 -14 24655 0 -14 3753.7999999999993 1 -14 23594 0 -5 3608.7902040790004 1 -5 10354 0 -15 3091.565714278 1 -15 21839 0 -13 3833.4 1 -13 21055 0 -5 968.3016326530001 1 -5 2021 0 -13 2714.6000000000004 1 -13 15584 0 -8 2603.9 1 -8 7485 0 -10 3057.5542857090004 1 -10 13378 0 -11 2758.2171428519996 1 -11 14533 0 -20 5824.184897953001 1 -20 54909 0 -14 4860.839183665999 1 -14 30433 0 -12 3022.863673464 1 -12 17076 0 -17 4378.5 1 -17 35996 0 -15 3921.8 1 -15 25440 0 -6 3638.3346938759996 1 -6 8866 0 diff --git a/music_album_creation/format_classification/test-split.txt b/music_album_creation/format_classification/test-split.txt deleted file mode 100644 index e42c4d5..0000000 --- a/music_album_creation/format_classification/test-split.txt +++ /dev/null @@ -1,78 +0,0 @@ -6 1758.1999999999998 1 -6 4225 0 -12 2855.5 1 -12 15575 0 -5 2626.1 1 -5 4932 0 -13 3642.5999999999995 1 -13 22758 0 -11 2800.169795913 1 -11 13557 0 -14 3927.2999999999997 1 -14 24669 0 -16 2867.7 1 -16 18485 0 -6 1661.2 1 -6 4006 0 -9 2547.1000000000004 1 -9 9571 0 -6 1257.4 1 -6 3247 0 -12 3578.2791836670003 1 -12 18717 0 -17 4592.483265297999 1 -17 37296 0 -11 2869.028571423 1 -11 13160 0 -4 1074.5902040810001 1 -4 1523 0 -11 3222.3 1 -11 16551 0 -14 2610.6 1 -14 16669 0 -8 2288.3004081589997 1 -8 7850 0 -6 1315.631020405 1 -6 3156 0 -10 2801.1 1 -10 11837 0 -12 2258.7000000000003 1 -12 11186 0 -10 2999.2489795870006 1 -10 13372 0 -9 2569.848163261 1 -9 9590 0 -6 2177.6 1 -6 4447 0 -17 4310.648163256999 1 -17 33314 0 -15 2663.8 1 -15 18042 0 -11 2172.212244893 1 -11 9830 0 -17 3348.8999999999996 1 -17 25841 0 -10 2532.414693873 1 -10 11267 0 -8 2337.279999996 1 -8 7021 0 -10 3071.687755098 1 -10 13023 0 -10 2662.922448973 1 -10 11644 0 -12 2913.5000000000005 1 -12 14897 0 -10 4091.3 1 -10 20064 0 -13 3017.8999999979997 1 -13 18315 0 -5 1282.690612243 1 -5 2338 0 -14 3654.6000000000004 1 -14 24330 0 -14 3573.315918361 1 -14 19656 0 -5 3095.7 1 -5 6377 0 -7 3496.7771428540004 1 -7 11241 0 diff --git a/music_album_creation/format_classification/train-split.txt b/music_album_creation/format_classification/train-split.txt deleted file mode 100644 index 5831160..0000000 --- a/music_album_creation/format_classification/train-split.txt +++ /dev/null @@ -1,608 +0,0 @@ -10 2556.5000000000005 1 -10 11563 0 -20 4111.595102032 1 -20 40624 0 -12 2259.4089795860004 1 -12 11497 0 -6 1844.245714285 1 -6 4787 0 -5 2505.6124081609996 1 -5 4492 0 -15 3595.9999999999995 1 -15 24650 0 -10 3175.9 1 -10 13598 0 -16 2814.3804081569997 1 -16 22520 0 -13 2463.3208163189997 1 -13 13425 0 -3 702.2 1 -3 727 0 -4 7012.900000000001 1 -4 4504 0 -3 1332.4 1 -3 1518 0 -12 2410.0 1 -12 12947 0 -10 2763.3 1 -10 11782 0 -11 2983.8240000000005 1 -11 14501 0 -9 2630.138775506 1 -9 9940 0 -12 3474.3 1 -12 17610 0 -13 2991.3999999999996 1 -13 17600 0 -4 904.2665306099999 1 -4 1337 0 -17 4253.1 1 -17 31829 0 -15 3660.3 1 -15 25234 0 -13 2639.542857137 1 -13 16765 0 -7 2200.3983673430002 1 -7 7106 0 -8 1692.6040816290001 1 -8 6197 0 -4 1049.1738775499998 1 -4 1645 0 -12 2709.054693872 1 -12 14259 0 -3 2385.024 1 -3 2177 0 -9 2510.367346934 1 -9 9639 0 -5 1467.944081632 1 -5 3012 0 -11 4551.523265301001 1 -11 23792 0 -13 2794.31836734 1 -13 16216 0 -12 2513.2 1 -12 13813 0 -6 1362.8 1 -6 3478 0 -10 3381.2 1 -10 12139 0 -13 2893.2 1 -13 17088 0 -6 3984.9 1 -6 8204 0 -5 1073.615918364 1 -5 2201 0 -5 2625.7240816310004 1 -5 4956 0 -5 3437.374693875 1 -5 4068 0 -10 2727.131428568 1 -10 11271 0 -14 2804.9 1 -14 17878 0 -18 4433.4236734589995 1 -18 36083 0 -16 2630.922448971 1 -16 18625 0 -5 1799.88897959 1 -5 3214 0 -9 2418.478367345 1 -9 10158 0 -10 2406.4722448949997 1 -10 10858 0 -7 1971.644081629 1 -7 5064 0 -7 1801.9265306080003 1 -7 5181 0 -10 1852.421224486 1 -10 7780 0 -12 2387.121632646 1 -12 13017 0 -13 2754.2 1 -13 14991 0 -10 2874.0179591779997 1 -10 10627 0 -13 2676.1926530539995 1 -13 15569 0 -6 3009.593469386 1 -6 7431 0 -8 3091.774693873 1 -8 10538 0 -4 881.5020408140001 1 -4 1324 0 -16 4261.6 1 -16 34131 0 -8 2080.032 1 -8 7212 0 -10 3115.0 1 -10 14125 0 -13 3982.576326526 1 -13 22111 0 -17 3411.5999999999995 1 -17 27264 0 -9 2645.4 1 -9 10905 0 -13 2942.171428565 1 -13 17595 0 -12 2533.8 1 -12 13085 0 -13 3726.9942857079996 1 -13 23494 0 -12 3178.9 1 -12 16282 0 -8 3885.1657142820004 1 -8 14567 0 -11 2723.892244892 1 -11 13817 0 -7 2231.37265306 1 -7 6859 0 -7 2832.7706122420004 1 -7 8681 0 -12 2899.382857136 1 -12 16043 0 -11 3817.5608163230004 1 -11 19886 0 -8 3040.3918367300003 1 -8 11619 0 -10 3398.269387751 1 -10 12848 0 -13 3351.5999999999995 1 -13 20134 0 -14 3656.2999999999997 1 -14 23500 0 -12 3482.8 1 -12 19060 0 -8 2267.350204077 1 -8 8262 0 -14 4467.983673461999 1 -14 25908 0 -10 2361.6 1 -10 9596 0 -12 2717.570612239 1 -12 15066 0 -8 2040.3999999999999 1 -8 6633 0 -7 1733.9559183650001 1 -7 4972 0 -13 3915.4155101980004 1 -13 20407 0 -5 1172.0 1 -5 2295 0 -8 2075.0 1 -8 6901 0 -5 1311.190204079 1 -5 2771 0 -4 2115.918367345 1 -4 4068 0 -5 1493.237714284 1 -5 2850 0 -11 2799.0000000000005 1 -11 13321 0 -15 4335.751836725999 1 -15 35686 0 -25 5903.999999999999 1 -25 70613 0 -11 4320.5 1 -11 12315 0 -19 4565.524897951001 1 -19 41457 0 -8 2582.4 1 -8 8488 0 -14 4150.171428564 1 -14 26334 0 -14 2627.604897953 1 -14 18032 0 -4 1007.9302040800001 1 -4 1371 0 -7 1528.111020405 1 -7 4382 0 -22 5326.561224485999 1 -22 56437 0 -10 3012.3999999999996 1 -10 10417 0 -11 3142.478367343 1 -11 15819 0 -12 3593.4 1 -12 20964 0 -14 4341.4 1 -14 28549 0 -12 4147.299999999999 1 -12 22958 0 -12 2419.7999999999997 1 -12 13219 0 -4 1265.2930612220002 1 -4 2048 0 -9 3300.9 1 -9 13570 0 -9 2530.682857142 1 -9 9841 0 -13 2928.5355101980003 1 -13 16252 0 -13 2966.5000000000005 1 -13 17411 0 -19 4772.783102031999 1 -19 44594 0 -17 3036.9 1 -17 24363 0 -14 3318.9 1 -14 19730 0 -11 2780.1861224429995 1 -11 11960 0 -7 2340.493061222 1 -7 7683 0 -10 3156.5 1 -10 12725 0 -7 1550.23673469 1 -7 4552 0 -11 2916.8 1 -11 14536 0 -23 6452.924081622 1 -23 76308 0 -14 3771.7999999999997 1 -14 24681 0 -13 3472.6999999999994 1 -13 20606 0 -12 2723.7 1 -12 14807 0 -10 2350.106122443 1 -10 10291 0 -14 3499.6 1 -14 23256 0 -9 3729.841632649 1 -9 14994 0 -15 3498.083265298 1 -15 24479 0 -17 4360.9 1 -17 35501 0 -16 2803.9 1 -16 20579 0 -6 1382.922448977 1 -6 3387 0 -12 3242.535510198 1 -12 18174 0 -12 2695.967346931 1 -12 14503 0 -18 3101.7273469290003 1 -18 24010 0 -5 1058.0 1 -5 1987 0 -5 2715.0367346909998 1 -5 5440 0 -8 2472.9 1 -8 7981 0 -6 3053.897142854 1 -6 7775 0 -13 3304.0 1 -13 18995 0 -12 2809.5999999999995 1 -12 15438 0 -12 3027.9 1 -12 17556 0 -14 3824.5 1 -14 24115 0 -15 4510.5 1 -15 31537 0 -6 2483.3306122409995 1 -6 6137 0 -11 2515.1 1 -11 12205 0 -11 2762.1 1 -11 13170 0 -10 2198.543673465 1 -10 9777 0 -10 2364.316734689 1 -10 9244 0 -8 3126.648163261 1 -8 10455 0 -6 2559.921632651 1 -6 7152 0 -11 3048.5000000000005 1 -11 15419 0 -10 3923.722448974 1 -10 17966 0 -7 2904.9469387719996 1 -7 7875 0 -12 3107.4481632590005 1 -12 16468 0 -4 2255.3 1 -4 4627 0 -4 2912.626938774 1 -4 4427 0 -11 2437.8999999999996 1 -11 12240 0 -16 4346.5 1 -16 29601 0 -15 2975.373061218 1 -15 20922 0 -10 3304.8 1 -10 14752 0 -6 1958.7657142829999 1 -6 4907 0 -7 2560.15673469 1 -7 7833 0 -12 3068.5 1 -12 16350 0 -10 4462.497959178 1 -10 14930 0 -15 4427.700000000001 1 -15 31379 0 -8 2021.433469383 1 -8 6785 0 -14 3667.4873469319996 1 -14 22660 0 -11 2918.9746938709995 1 -11 14514 0 -5 2856.489795916 1 -5 4857 0 -13 2441.1000000000004 1 -13 13853 0 -3 2005.0319999999997 1 -3 2009 0 -8 2825.09061224 1 -8 7673 0 -18 4556.643265296 1 -18 36477 0 -7 1383.3926530570002 1 -7 4204 0 -13 2358.7000000000003 1 -13 13754 0 -13 4737.8024489730005 1 -13 22713 0 -6 1374.262857141 1 -6 3308 0 -9 3082.422857139 1 -9 12319 0 -13 2939.8465306070007 1 -13 17694 0 -17 4715.546122441 1 -17 34816 0 -10 2347.3999999999996 1 -10 9771 0 -5 2116.9893877520003 1 -5 4244 0 -8 1945.6595918360001 1 -8 7102 0 -12 3632.6 1 -12 19254 0 -8 1923.395918363 1 -8 6540 0 -10 2659.082448975 1 -10 11778 0 -12 3601.0318367299997 1 -12 18680 0 -15 3866.253061217 1 -15 26604 0 -15 3293.8999999999996 1 -15 22304 0 -14 3061.1 1 -14 22052 0 -12 3600.7 1 -12 16690 0 -10 2936.24163265 1 -10 11314 0 -11 2661.1920000000005 1 -11 13445 0 -13 3686.7999999999997 1 -13 20132 0 -11 2483.226122443 1 -11 11557 0 -12 2423.013877547 1 -12 12757 0 -5 3548.238367344 1 -5 7290 0 -14 3314.6514285649996 1 -14 19607 0 -7 2144.1306122409997 1 -7 6484 0 -8 2463.6 1 -8 7413 0 -9 2604.1469387720003 1 -9 10100 0 -18 4824.599999999999 1 -18 45983 0 -11 2926.2 1 -11 14239 0 -14 3074.324897952 1 -14 20185 0 -6 2290.4424489760004 1 -6 5676 0 -6 1626.592653058 1 -6 4186 0 -10 2937.1820408099998 1 -10 12198 0 -7 1613.0 1 -7 4823 0 -12 2829.7000000000003 1 -12 14792 0 -13 4036.8326530540007 1 -13 24974 0 -12 2886.7 1 -12 15500 0 -3 2930.4 1 -3 4616 0 -9 2994.0 1 -9 11411 0 -13 4377.024 1 -13 22733 0 -15 3372.199183665 1 -15 23384 0 -4 2110.350204081 1 -4 2525 0 -22 4097.6195918270005 1 -22 37647 0 -19 3933.675102032 1 -19 36232 0 -13 2171.3 1 -13 12455 0 -5 1151.973877549 1 -5 2133 0 -11 2364.7 1 -11 12307 0 -16 3769.782857135 1 -16 28825 0 -4 3806.7722448960003 1 -4 5579 0 -5 1365.317142856 1 -5 2965 0 -13 3139.2914285660004 1 -13 18548 0 -10 3239.1 1 -10 12731 0 -8 2283.755102036 1 -8 7944 0 -11 2228.7999999999997 1 -11 11000 0 -14 3759.4644897889993 1 -14 23668 0 -4 901.615510204 1 -4 1394 0 -11 2350.3457142790003 1 -11 12079 0 -8 1942.622040813 1 -8 6624 0 -12 3725.6359183619998 1 -12 16519 0 -4 925.0 1 -4 1315 0 -15 3982.8999999999996 1 -15 28104 0 -8 2243.0040816279998 1 -8 7344 0 -14 2900.8979591769994 1 -14 17624 0 -12 2888.5681632610003 1 -12 14749 0 -6 1644.9999999999998 1 -6 3904 0 -3 1671.7322448959999 1 -3 1356 0 -6 1614.7 1 -6 3922 0 -9 2606.2999999999997 1 -9 9372 0 -18 4432.0 1 -18 32606 0 -10 2483.6 1 -10 10517 0 -7 2409.508571426 1 -7 7143 0 -11 2963.2 1 -11 13917 0 -15 3426.455510196 1 -15 22397 0 -10 2517.8 1 -10 11262 0 -4 1249.2277551000002 1 -4 1933 0 -11 3535.4383673429998 1 -11 18706 0 -7 1692.7869387709998 1 -7 5428 0 -6 2363.742040814 1 -6 5655 0 -5 2435.016 1 -5 4950 0 -5 968.933877548 1 -5 2218 0 -4 2011.6080000000002 1 -4 3457 0 -10 2423.381877548 1 -10 11869 0 -5 923.689795917 1 -5 1765 0 -15 3908.2999999999997 1 -15 26637 0 -13 3663.2 1 -13 17404 0 -14 3635.0955101969994 1 -14 23754 0 -6 2520.3199999969997 1 -6 6577 0 -14 4057.417142851 1 -14 27028 0 -14 3799.8 1 -14 24559 0 -9 1738.7885714249996 1 -9 6998 0 -16 2984.28081632 1 -16 21542 0 -6 2510.016 1 -6 5635 0 -15 2910.249795911 1 -15 21342 0 -6 2171.846530609 1 -6 5492 0 -16 2902.2000000000003 1 -16 22439 0 -6 1460.2 1 -6 4051 0 -6 1535.88 1 -6 3759 0 -3 688.5 1 -3 722 0 -3 899.81510204 1 -3 915 0 -8 2294.5 1 -8 8248 0 -10 2880.7999999999997 1 -10 13177 0 -9 2161.397551017 1 -9 9034 0 -8 2076.238367344 1 -8 8202 0 -14 2619.5 1 -14 15203 0 -8 2737.737142854 1 -8 8535 0 -12 4737.5 1 -12 24801 0 -13 3406.5 1 -13 16735 0 -38 6189.400816308001 1 -38 108396 0 -6 2440.032 1 -6 5658 0 -12 2716.3999999999996 1 -12 14971 0 -12 3429.6163265239993 1 -12 20305 0 -4 1159.8 1 -4 1722 0 -9 3421.2310204050004 1 -9 12125 0 -6 2330.932244895 1 -6 5909 0 -12 3075.0 1 -12 17562 0 -13 2822.608979585 1 -13 16366 0 -15 4644.499999999999 1 -15 29941 0 -15 3288.2999999999997 1 -15 23015 0 -7 2340.884897955 1 -7 6658 0 -12 3088.0130612189996 1 -12 16545 0 -12 2638.5000000000005 1 -12 15098 0 -12 2484.8 1 -12 13975 0 -7 1860.75428571 1 -7 5579 0 -5 1133.400816324 1 -5 2087 0 -15 3122.8 1 -15 21462 0 -14 3681.000000000001 1 -14 23910 0 -4 1027.118367346 1 -4 1677 0 -6 1414.086530608 1 -6 3367 0 -5 2098.729795915 1 -5 3392 0 -4 1175.2 1 -4 1906 0 -14 2891.3999999999996 1 -14 16302 0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..9ad0981 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[metadata] +description-file = README.rst +license_file = LICENSE.txt diff --git a/setup.py b/setup.py index 6696d8a..329eb52 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ import os from setuptools import setup, find_packages + + my_dir = os.path.dirname(os.path.realpath(__file__)) def readme(): @@ -9,8 +11,8 @@ def readme(): setup( - name='music_album_creator', - version='1.0.7', + name='music_album_creation', + version='1.0.8a', description='A CLI application intending to automate offline music library building', long_description=readme(), keywords='music album automation youtube audio metadata download', @@ -30,11 +32,14 @@ def readme(): author='Konstantinos Lampridis', author_email='k.lampridis@hotmail.com', license='GNU GPLv3', - packages=find_packages(exclude=["testing.*", "testing"]), + packages=find_packages(where='src'), + package_dir={'':'src'}, install_requires=['tqdm', 'click', 'sklearn', 'mutagen', 'PyInquirer', 'youtube_dl'], include_package_data=True, - entry_points = { - 'console_scripts': ['create-album=music_album_creation.create_album:main'], + entry_points={ + 'console_scripts': [ + 'create-album = music_album_creation.create_album:main', + ] }, setup_requires=['pytest-runner>=2.0',], tests_require=['pytest',], diff --git a/src/music_album_creation/__init__.py b/src/music_album_creation/__init__.py new file mode 100644 index 0000000..c545e69 --- /dev/null +++ b/src/music_album_creation/__init__.py @@ -0,0 +1,8 @@ +__version__ = '1.0.8a' + +from .tracks_parsing import StringParser +from .metadata import MetadataDealer +from .format_classification import FormatClassifier + +from .downloading import YoutubeDownloader +from .album_segmentation import AudioSegmenter diff --git a/music_album_creation/album_segmentation.py b/src/music_album_creation/album_segmentation.py similarity index 100% rename from music_album_creation/album_segmentation.py rename to src/music_album_creation/album_segmentation.py diff --git a/music_album_creation/create_album.py b/src/music_album_creation/create_album.py similarity index 88% rename from music_album_creation/create_album.py rename to src/music_album_creation/create_album.py index 8a087b0..4baf9b2 100644 --- a/music_album_creation/create_album.py +++ b/src/music_album_creation/create_album.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import re import os @@ -11,15 +11,13 @@ import subprocess from time import sleep -from tracks_parsing import parser -from metadata import MetadataDealer -from format_classification import FormatClassifier -from downloading import YoutubeDownloader as youtube -from album_segmentation import AudioSegmenter, TrackTimestampsSequenceError, WrongTimestampFormat -from downloading import TokenParameterNotInVideoInfoError, InvalidUrlError, UnavailableVideoError +from . import StringParser, MetadataDealer, AudioSegmenter, FormatClassifier +from .tracks_parsing import TrackTimestampsSequenceError, WrongTimestampFormat + +from .downloading import TokenParameterNotInVideoInfoError, InvalidUrlError, UnavailableVideoError # 'front-end', interface, interactive dialogs are imported below -from dialogs import track_information_type_dialog, interactive_track_info_input_dialog, \ +from .dialogs import track_information_type_dialog, interactive_track_info_input_dialog, \ store_album_dialog, interactive_metadata_dialogs, input_youtube_url_dialog, update_and_retry_dialog @@ -58,13 +56,13 @@ def main(tracks_info, track_name, track_number, artist, album_artist, url): done = False while not done: try: - youtube.download(video_url, directory, spawn=False, verbose=True, supress_stdout=False) # force waiting before continuing execution, by not spawning a separate process + YoutubeDownloader.download(video_url, directory, spawn=False, verbose=True, supress_stdout=False) # force waiting before continuing execution, by not spawning a separate process done = True except TokenParameterNotInVideoInfoError as e: print(e, '\n') if update_and_retry_dialog()['update-youtube-dl']: - print("About to execute '{}'".format(youtube.update_backend_command)) - ro = youtube.update_backend() + print("About to execute '{}'".format(YoutubeDownloader.update_backend_command)) + ro = YoutubeDownloader.update_backend() else: print("Exiting ..") sys.exit(1) diff --git a/music_album_creation/dialogs.py b/src/music_album_creation/dialogs.py similarity index 100% rename from music_album_creation/dialogs.py rename to src/music_album_creation/dialogs.py diff --git a/music_album_creation/display-logo.sh b/src/music_album_creation/display-logo.sh similarity index 100% rename from music_album_creation/display-logo.sh rename to src/music_album_creation/display-logo.sh diff --git a/music_album_creation/downloading.py b/src/music_album_creation/downloading.py similarity index 100% rename from music_album_creation/downloading.py rename to src/music_album_creation/downloading.py diff --git a/music_album_creation/format_classification/__init__.py b/src/music_album_creation/format_classification/__init__.py similarity index 100% rename from music_album_creation/format_classification/__init__.py rename to src/music_album_creation/format_classification/__init__.py diff --git a/music_album_creation/format_classification/data/dev-split.txt b/src/music_album_creation/format_classification/data/dev-split.txt similarity index 100% rename from music_album_creation/format_classification/data/dev-split.txt rename to src/music_album_creation/format_classification/data/dev-split.txt diff --git a/music_album_creation/format_classification/data/model.pickle b/src/music_album_creation/format_classification/data/model.pickle similarity index 100% rename from music_album_creation/format_classification/data/model.pickle rename to src/music_album_creation/format_classification/data/model.pickle diff --git a/music_album_creation/format_classification/data/model1 b/src/music_album_creation/format_classification/data/model1 similarity index 100% rename from music_album_creation/format_classification/data/model1 rename to src/music_album_creation/format_classification/data/model1 diff --git a/music_album_creation/format_classification/data/test-split.txt b/src/music_album_creation/format_classification/data/test-split.txt similarity index 100% rename from music_album_creation/format_classification/data/test-split.txt rename to src/music_album_creation/format_classification/data/test-split.txt diff --git a/music_album_creation/format_classification/data/train-split.txt b/src/music_album_creation/format_classification/data/train-split.txt similarity index 100% rename from music_album_creation/format_classification/data/train-split.txt rename to src/music_album_creation/format_classification/data/train-split.txt diff --git a/music_album_creation/format_classification/dataset.py b/src/music_album_creation/format_classification/dataset.py similarity index 100% rename from music_album_creation/format_classification/dataset.py rename to src/music_album_creation/format_classification/dataset.py diff --git a/music_album_creation/format_classification/tracks_format_classifier.py b/src/music_album_creation/format_classification/tracks_format_classifier.py similarity index 100% rename from music_album_creation/format_classification/tracks_format_classifier.py rename to src/music_album_creation/format_classification/tracks_format_classifier.py diff --git a/music_album_creation/metadata.py b/src/music_album_creation/metadata.py similarity index 100% rename from music_album_creation/metadata.py rename to src/music_album_creation/metadata.py diff --git a/music_album_creation/tracks_parsing.py b/src/music_album_creation/tracks_parsing.py similarity index 100% rename from music_album_creation/tracks_parsing.py rename to src/music_album_creation/tracks_parsing.py diff --git a/testing/__init__.py b/testing/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/music_album_creation/__init__.py b/tests/__init__.py similarity index 100% rename from music_album_creation/__init__.py rename to tests/__init__.py diff --git a/testing/know_your_enemy.mp3 b/tests/know_your_enemy.mp3 similarity index 100% rename from testing/know_your_enemy.mp3 rename to tests/know_your_enemy.mp3 diff --git a/testing/test_classifier.py b/tests/test_classifier.py similarity index 81% rename from testing/test_classifier.py rename to tests/test_classifier.py index f274fad..873c97f 100644 --- a/testing/test_classifier.py +++ b/tests/test_classifier.py @@ -1,10 +1,12 @@ import os import pytest +import music_album_creation + from music_album_creation.format_classification import dataset_handler, FormatClassifier -model = "music_album_creation/format_classification/data/model.pickle" +model = "src/music_album_creation/format_classification/data/model.pickle" @pytest.fixture(scope='module') diff --git a/tests/test_create_album_program.py b/tests/test_create_album_program.py new file mode 100644 index 0000000..6e0c5c9 --- /dev/null +++ b/tests/test_create_album_program.py @@ -0,0 +1,19 @@ + +import subprocess +import pytest + +from click.testing import CliRunner +import music_album_creation + +from music_album_creation.create_album import main + +class TestCreateAlbum: + + def test_launching(self): + ro = subprocess.run(['create-album', '--help'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert ro.returncode == 0 + + def test_main(self): + runner = CliRunner() + result = runner.invoke(main, ['--help']) + assert result.exit_code == 0 diff --git a/testing/test_downloading.py b/tests/test_downloading.py similarity index 97% rename from testing/test_downloading.py rename to tests/test_downloading.py index 7059867..afaeb5f 100644 --- a/testing/test_downloading.py +++ b/tests/test_downloading.py @@ -1,6 +1,7 @@ import os import pytest +import music_album_creation from music_album_creation.downloading import YoutubeDownloader, InvalidUrlError, UnavailableVideoError diff --git a/testing/test_segmenting.py b/tests/test_segmenting.py similarity index 99% rename from testing/test_segmenting.py rename to tests/test_segmenting.py index b1c1ab4..d2eee17 100644 --- a/testing/test_segmenting.py +++ b/tests/test_segmenting.py @@ -2,6 +2,7 @@ import pytest import mutagen +import music_album_creation from music_album_creation.album_segmentation import AudioSegmenter, NotStartingFromZeroTimestampError from music_album_creation.tracks_parsing import WrongTimestampFormat, TrackTimestampsSequenceError diff --git a/testing/test_splitters.py b/tests/test_splitters.py similarity index 97% rename from testing/test_splitters.py rename to tests/test_splitters.py index 5e74561..d6e0c3e 100644 --- a/testing/test_splitters.py +++ b/tests/test_splitters.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import pytest - +import music_album_creation from music_album_creation.tracks_parsing import StringParser From 4b5ac984d89c272e5b65cca73bbef521e833000a Mon Sep 17 00:00:00 2001 From: Konstantinos <> Date: Tue, 9 Jul 2019 23:16:53 +0200 Subject: [PATCH 2/9] Fix references --- src/music_album_creation/create_album.py | 10 +++++----- src/music_album_creation/tracks_parsing.py | 2 +- tests/test_splitters.py | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/music_album_creation/create_album.py b/src/music_album_creation/create_album.py index 4baf9b2..e6d3a2e 100644 --- a/src/music_album_creation/create_album.py +++ b/src/music_album_creation/create_album.py @@ -14,7 +14,7 @@ from . import StringParser, MetadataDealer, AudioSegmenter, FormatClassifier from .tracks_parsing import TrackTimestampsSequenceError, WrongTimestampFormat -from .downloading import TokenParameterNotInVideoInfoError, InvalidUrlError, UnavailableVideoError +from .downloading import YoutubeDownloader, TokenParameterNotInVideoInfoError, InvalidUrlError, UnavailableVideoError # 'front-end', interface, interactive dialogs are imported below from .dialogs import track_information_type_dialog, interactive_track_info_input_dialog, \ @@ -77,7 +77,7 @@ def main(tracks_info, track_name, track_number, artist, album_artist, url): print('\n') album_file = os.path.join(directory, os.listdir(directory)[0]) - guessed_info = parser.parse_album_info(album_file) + guessed_info = StringParser.parse_album_info(album_file) audio_segmenter = AudioSegmenter(target_directory=directory) ### RECEIVE TRACKS INFORMATION @@ -89,7 +89,7 @@ def main(tracks_info, track_name, track_number, artist, album_artist, url): tracks_string = interactive_track_info_input_dialog().strip() print() try: - tracks_data = parser.parse_hhmmss_string(tracks_string) + tracks_data = StringParser.parse_hhmmss_string(tracks_string) except WrongTimestampFormat as e: print(e) sys.exit(1) @@ -101,7 +101,7 @@ def main(tracks_info, track_name, track_number, artist, album_artist, url): answer = track_information_type_dialog(prediction={1:'durations'}.get(int(predicted_label), 'timestamps')) if answer.startswith('Durations'): - tracks_data = parser.duration_data_to_timestamp_data(tracks_data) + tracks_data = StringParser.duration_data_to_timestamp_data(tracks_data) try: audio_files = audio_segmenter.segment_from_list(album_file, tracks_data, supress_stdout=True, verbose=True, sleep_seconds=0.4) except TrackTimestampsSequenceError as e: @@ -110,7 +110,7 @@ def main(tracks_info, track_name, track_number, artist, album_artist, url): # TODO capture ctrl-D to signal possible change of type from timestamp to durations and vice-versa... # in order to put the above statement outside of while loop - durations = [parser.time_format(getattr(mutagen.File(os.path.join(directory, t)).info, 'length', 0)) for t in audio_files] + durations = [StringParser.time_format(getattr(mutagen.File(os.path.join(directory, t)).info, 'length', 0)) for t in audio_files] max_row_length = max(len(_[0]) + len(_[1]) for _ in zip(audio_files, durations)) print("\n\nThese are the tracks created.\n".format(os.path.dirname(audio_files[0]))) print('\n'.join(sorted([' {}{} {}'.format(t, (max_row_length - len(t) - len(d)) * ' ', d) for t, d in zip(audio_files, durations)])), '\n') diff --git a/src/music_album_creation/tracks_parsing.py b/src/music_album_creation/tracks_parsing.py index e94e78b..f5f5c8e 100644 --- a/src/music_album_creation/tracks_parsing.py +++ b/src/music_album_creation/tracks_parsing.py @@ -60,7 +60,7 @@ def _parse_string(cls, tracks): def _parse_track_line(cls, track_line): """Parses a string line such as '01. Doteru 3:45'""" regex = re.compile(r"""^(?:\d{1,2}(?:[\ \t]*[\.\-,][\ \t]*|[\t\ ]+))? # potential track number (eg 01) included is ignored - ([\w\'\(\) ]*[\w)]) # track name + ([\w\'\(\) ’]*[\w)]) # track name (?:[\t ]+|[\t ]*[\-\.]+[\t ]*) # separator between name and time ((?:\d?\d:)*\d?\d)$ # time in hh:mm:ss format""", re.X) # regex = re.compile('^(?:\d{1,2}([\ \t]*[\.\-,][ \t]*|[\t ]+))?([\w\'\(\) ]*[\w)])' + cls.sep + '((?:\d?\d:)*\d?\d)$') diff --git a/tests/test_splitters.py b/tests/test_splitters.py index d6e0c3e..5ef3744 100644 --- a/tests/test_splitters.py +++ b/tests/test_splitters.py @@ -23,3 +23,17 @@ def test_tracks_line_parsing(self, track_line, name, time): ]) def test_youtube_video_title_parsing(self, video_title, artist, album, year): assert StringParser.parse_album_info(video_title) == {'artist': artist, 'album': album, 'year': year} + + @pytest.mark.parametrize("tracks_string", [ + ('1. Virtual Funeral - 0:00\n2. Macedonian Lines - 6:46\n3. Melancholy Sadie - 11:30\n4. Bowie’s Last Breath - 16:19\n5. I’m Not A Real Indian (But I Play One On TV) - 20:20\n6. I Make Weird Choices - 23:44'), + ('1. Virtual Funeral - 0:00\n2. Macedonian Lines - 6:46\n3. Melancholy Sadie - 11:30\n4. Bowie’s Last Breath - 16:19\n5. I’m Not A Real Indian (But I Play One On TV) - 20:20\n6. I Make Weird Choices - 23:44\n'), + ('1 Virtual Funeral - 0:00\n2 Macedonian Lines - 6:46\n3 Melancholy Sadie - 11:30\n4 Bowie’s Last Breath - 16:19\n5 I’m Not A Real Indian (But I Play One On TV) - 20:20\n6 I Make Weird Choices - 23:44'), + ('1 Virtual Funeral - 0:00\n2 Macedonian Lines - 6:46\n3 Melancholy Sadie - 11:30\n4 Bowie’s Last Breath - 16:19\n5 I’m Not A Real Indian (But I Play One On TV) - 20:20\n6 I Make Weird Choices - 23:44\n'), + ]) + def test_tracks_string(self, tracks_string): + assert StringParser.parse_hhmmss_string(tracks_string) == [['Virtual Funeral', '0:00'], + ['Macedonian Lines', '6:46'], + ['Melancholy Sadie', '11:30'], + ['Bowie’s Last Breath', '16:19'], + ["I’m Not A Real Indian (But I Play One On TV)", '20:20'], + ['I Make Weird Choices', '23:44']] \ No newline at end of file From d5f85a2593ec5e9c7d44490793453a8bc24bdc1c Mon Sep 17 00:00:00 2001 From: Konstantinos <> Date: Wed, 10 Jul 2019 13:56:45 +0200 Subject: [PATCH 3/9] Fix prediction output string, change regex of MetadataDealer to parse track file names with extra special character and add some comments --- src/music_album_creation/create_album.py | 3 ++- src/music_album_creation/dialogs.py | 4 ++-- src/music_album_creation/metadata.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/music_album_creation/create_album.py b/src/music_album_creation/create_album.py index e6d3a2e..8329f96 100644 --- a/src/music_album_creation/create_album.py +++ b/src/music_album_creation/create_album.py @@ -88,6 +88,7 @@ def main(tracks_info, track_name, track_number, artist, album_artist, url): sleep(0.50) tracks_string = interactive_track_info_input_dialog().strip() print() + # Convert string with tracks and timestamps information to data structure try: tracks_data = StringParser.parse_hhmmss_string(tracks_string) except WrongTimestampFormat as e: @@ -102,7 +103,7 @@ def main(tracks_info, track_name, track_number, artist, album_artist, url): if answer.startswith('Durations'): tracks_data = StringParser.duration_data_to_timestamp_data(tracks_data) - try: + try: # SEGMENTATION audio_files = audio_segmenter.segment_from_list(album_file, tracks_data, supress_stdout=True, verbose=True, sleep_seconds=0.4) except TrackTimestampsSequenceError as e: print(e) diff --git a/src/music_album_creation/dialogs.py b/src/music_album_creation/dialogs.py index b866d63..eff6d8f 100644 --- a/src/music_album_creation/dialogs.py +++ b/src/music_album_creation/dialogs.py @@ -23,14 +23,14 @@ def update_and_retry_dialog(): 'default': True, } ] - answer = promt(questions) + answer = prompt(questions) return answer ##### MULTILINE INPUT TRACK NAMES AND TIMESTAMPS (hh:mm:ss) def track_information_type_dialog(prediction=''): """Returns a parser of track hh:mm:ss multiline string""" - if prediction == 'Timestamps': + if prediction == 'timestamps': choices = ['Timestamps (predicted)', 'Durations'] elif prediction == 'durations': choices = ['Durations (predicted)', 'Timestamps'] diff --git a/src/music_album_creation/metadata.py b/src/music_album_creation/metadata.py index 7b9fb38..f98a093 100644 --- a/src/music_album_creation/metadata.py +++ b/src/music_album_creation/metadata.py @@ -48,7 +48,7 @@ class MetadataDealer(metaclass=MetadataDealerType): _all = dict(_d, **dict(_auto_data)) - reg = re.compile(r'(?:(\d{1,2})(?:[ \t]*[\-\.][ \t]*|[ \t]+)|^)?((?:\w+\b[ \t])*?\w+)(?:\.\w+)') # use to parse track file names like "1. Loyal to the Pack.mp3" + reg = re.compile(r'(?:(\d{1,2})(?:[ \t]*[\-\.][ \t]*|[ \t]+)|^)?([\w\'\(\) ’]*[\w)])\.mp3$') # use to parse track file names like "1. Loyal to the Pack.mp3" def set_album_metadata(self, album_directory, track_number=True, track_name=True, artist='', album_artist='', album='', year='', verbose=False): self._write_metadata(album_directory, track_number=track_number, track_name=track_name, artist=artist, From 15103be6b169b09d44c86b96e15c75c501d5deb3 Mon Sep 17 00:00:00 2001 From: Konstantinos <> Date: Wed, 10 Jul 2019 16:49:02 +0200 Subject: [PATCH 4/9] Configure tox.ini with various python interpreters for various environments which run both unittests, lint (flake8) code style test and source distribution files-check. Configure tox to use pytest, pytest-conv and invoke the last, which runs pytest and coverage. Comply with flake8 style warnings. Configure travis to run parallel jobs for environments py35, py36, p37 and 'quality' (distro-check and flake8). All py3\d jobs send coverage data to coveralls.io automatically at the end of execution. Instruct travis to run tox as a testing tool and indicate the desired environments to initialize Instruct through setup.cfg what arguments the pytest command will utilize (when executed by tox) Change setup.py to include th 'py_modules' parameter in setup callback and remove 'pytest-runner>=2.0' from 'setup_requires' list Add AUTHORS.rst file Track tox.ini file Update MANIFEST to push 'requirements.txt', '.coveragerc', 'tox.ini' files and '.travis/' directory and subfiles to in the source distribution Update MANIFEST to push 'requirements.txt', '.coveragerc', 'tox.ini' files and '.travis/' directory and subfiles to in the source distribution WIP WIP Add renamed file WIP WIP Remove couple of print statements from TabCompleter Comply with flake8 warnings Fix import order in format_classification.__init__ Remove orphan travis test for python3.6 interpreter Complete 'flake8' config in tox.ini WIP WIP Pass all build tests for all environemnts of tox Enable travis jobs for 'report' and 'coveralls' build tests Try enabling 'stages' for travis WIP WIP WIP WIP WIP WIP WIP WIP WIP Install tox-travis for travis jobs Rewrite condition for deployment script WIP Enable deployment script for all branches Try build stages feature and tox combo WIP WIP WIP WIP WIP WIP WIP Attemp1 Attemp2 Attemp3 Attemp4 Attemp5 Attemp6 Attemp7 Attemp8 Attemp9 Attemp9 Attemp10 Run put coveralls invocation in .travis/run-tox.sh script Pip isntall coveralls in 'travis.install' and remove Clean stage Ensure that, in case of interpreter-missing error, travis will fail instead of the 'local' configuration where the test is skipped --- .coveragerc | 16 ++ .travis.yml | 153 +++++++++++++++- .travis/deploy.sh | 3 + .travis/install-tox.sh | 6 + .travis/run-tox.sh | 10 ++ AUTHORS.rst | 5 + MANIFEST.in | 5 + README.rst | 29 +-- setup.cfg | 35 ++++ setup.py | 18 +- src/music_album_creation/__init__.py | 2 + .../album_segmentation.py | 2 - src/music_album_creation/create_album.py | 16 +- src/music_album_creation/dialogs.py | 6 +- src/music_album_creation/downloading.py | 6 +- .../format_classification/__init__.py | 4 +- .../format_classification/dataset.py | 24 ++- .../tracks_format_classifier.py | 2 +- src/music_album_creation/metadata.py | 19 +- src/music_album_creation/tracks_parsing.py | 18 +- tests/test_classifier.py | 3 - tests/test_create_album_program.py | 3 +- tests/test_downloading.py | 3 +- tests/test_segmenting.py | 7 +- tests/test_splitters.py | 3 +- tox.ini | 165 ++++++++++++++++++ 26 files changed, 470 insertions(+), 93 deletions(-) create mode 100644 .coveragerc create mode 100644 .travis/deploy.sh create mode 100644 .travis/install-tox.sh create mode 100644 .travis/run-tox.sh create mode 100644 AUTHORS.rst create mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..19e5ee1 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,16 @@ +[paths] +source = + src + */site-packages + +[run] +branch = true +source = + music_album_creator + tests +parallel = true + +[report] +show_missing = true +precision = 2 +omit = *migrations* diff --git a/.travis.yml b/.travis.yml index 72285f9..5cafbdd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,151 @@ language: python -python: - - "3.5" - - "3.6" - - "3.7" +env: + global: + - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so + - SEGFAULT_SIGNALS=all + +#matrix: +# include: +# - python: '3.6' +# env: +# - TOXENV=check +# - env: +# - TOXENV=py35 +# python: '3.5' +# - env: +# - TOXENV=py36 +# python: '3.6' +# - env: +# - TOXENV=py37 +# python: '3.7' + + before_install: + - python --version +# - pip install coveralls + - uname -a + - lsb_release -a + - chmod +x .travis/install-tox.sh + - chmod +x .travis/run-tox.sh - sudo apt-get install ffmpeg +# - chmod +x .travis/deploy.sh + install: - - pip install . -script: python -m pytest \ No newline at end of file + - .travis/install-tox.sh +# - pip install tox-travis + - virtualenv --version + - easy_install --version + - pip --version + - tox --version + - pip install coveralls + +cache: pip +script: .travis/run-tox.sh + +jobs: + fail_fast: true + include: +# - stage: clean coverage data +# python: '3.6' +# env: TOXENV=clean + - stage: run unittests + python: '3.6' + env: TOXENV=quality + script: + - export TOX_SKIP_MISSING_INTERPRETERS="False" + - tox + - stage: run unittests + python: '3.5' + env: TOXENV=py35 + - stage: run unittests + python: '3.6' + env: TOXENV=py36 + - stage: run unittests + python: '3.7' + env: TOXENV=py37 +# - stage: send coverage data to coveralls.io +# python: '3.6' +# env: TOXENV=reporting +# allow_failures: +# - env: TOXENV=reporting + +after_failure: + - more .tox/log/* | cat + - more .tox/*/log/* | cat + +# - stage: run tests and linters +# language: node_js +# node_js: 6 +# cache: yarn +# env: COMPONENT=server CMD=lint +# before_install: cd server +# script: bash ../scripts/travis-yarn.sh +# +#quality: &quality +# - stage: "Quality" +# name: "Quallity assertion" +# script: .travis/run-tox.sh +# +#tests: &tests +# - stage: "Tests" +# name: "Unit Tests" +# script: .travis/run-tox.sh + + +# - name: "Helper Tests" +# script: time ./run yarn test:named Helper +# - name: "Integration Tests" +# script: time ./run yarn test:named Integration +# - name: "Acceptance Tests" +# script: time ./run yarn test:named Acceptance +# - name: "a11y" +# script: PERCY_ENABLE=0 time ./run yarn test:named Acceptance --query enableA11yAudit=true + +#quality: &quality +# - stage: "Quality" +# name: "Lint JS/TS" +# script: time ./run yarn lint:js +# - name: "Lint Templates" +# script: time ./run yarn lint:hbs +# - name: "Lint Styles" +# script: time ./run yarn lint:sass +# - name: "Check Types" +# script: time ./run yarn tsc +# - name: "Translations" +# script: time ./run yarn lint:i18n + +#jobs: +# fail_fast: true +# +# include: +# - <<: *quality +# env: +# - TOXENV=quality + +# - <<: *tests +# env: +# - TOXENV=py35 +## +# - <<: *tests +# python: 3.6 +# env: +# - TOXENV=py36 +# +# - <<: *tests +# python: 3.7 +# env: +# - TOXENV=py37 + +#script: ./.travis/run-tox.sh +# +#deploy: +# provider: script +# script: .travis/deploy.sh +# on: +# all_branches: true + + +#notifications: +# email: +# on_success: never +# on_failure: never diff --git a/.travis/deploy.sh b/.travis/deploy.sh new file mode 100644 index 0000000..e6af562 --- /dev/null +++ b/.travis/deploy.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +coveralls \ No newline at end of file diff --git a/.travis/install-tox.sh b/.travis/install-tox.sh new file mode 100644 index 0000000..82ef97f --- /dev/null +++ b/.travis/install-tox.sh @@ -0,0 +1,6 @@ +!#/bin/bash + +set -e +set -u + +pip install tox diff --git a/.travis/run-tox.sh b/.travis/run-tox.sh new file mode 100644 index 0000000..17ef343 --- /dev/null +++ b/.travis/run-tox.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e +set -u + +export TOX_SKIP_MISSING_INTERPRETERS="False"; + +tox + +coveralls \ No newline at end of file diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000..7ba338e --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,5 @@ + +Authors +======= + +* Konstantinos Lampridis - https://github.com/boromir674 diff --git a/MANIFEST.in b/MANIFEST.in index 0ceabd4..a271bea 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include AUTHORS.rst include CHANGELOG.rst include LICENSE.txt +include requirements.txt include src/music_album_creation/format_classification/data/* include src/music_album_creation/display-logo.sh @@ -12,5 +13,9 @@ graft src graft tests include .travis.yml +include .coveragerc +include tox.ini + +graft .travis global-exclude *.py[cod] __pycache__ *.so *.dylib diff --git a/README.rst b/README.rst index 86b5006..c6b132a 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ Music Album Creator - CLI Application ===================================== -Music Album Creator is a cli application aiming to automate the process of building an offline music library. +Music Album Creator is a cli application aiming to automate the process of building an offline music digital library. ======== @@ -19,8 +19,8 @@ Overview - | |travis| | |coveralls| * - package - - | |version| |wheel| |supported-versions| |supported-implementations| - | |commits-since| + - | |version| |wheel| + .. |docs| image:: https://readthedocs.org/projects/music-album-creator/badge/?style=flat :target: https://readthedocs.org/projects/music-album-creator :alt: Documentation Status @@ -29,9 +29,9 @@ Overview :alt: Travis-CI Build Status :target: https://travis-ci.org/boromir674/music-album-creator -.. |coveralls| image:: https://coveralls.io/repos/boromir674/music-album-creator/badge.svg?branch=master&service=github +.. |coveralls| image:: https://coveralls.io/repos/github/boromir674/music-album-creator/badge.svg?branch=dev :alt: Coverage Status - :target: https://coveralls.io/r/boromir674/music-album-creator + :target: https://coveralls.io/github/boromir674/music-album-creator?branch=dev .. |version| image:: https://img.shields.io/pypi/v/music-album-creator.svg :alt: PyPI Package latest release @@ -58,7 +58,7 @@ Overview A CLI application intending to automate offline music library building. -* Free software: Apache Software License 2.0 +* Free software: GNU General Public License v3.0 Installation ============ @@ -80,20 +80,3 @@ Development To run the all tests run:: tox - -Note, to combine the coverage data from all the tox environments run: - -.. list-table:: - :widths: 10 90 - :stub-columns: 1 - - - - Windows - - :: - - set PYTEST_ADDOPTS=--cov-append - tox - - - - Other - - :: - - PYTEST_ADDOPTS=--cov-append tox diff --git a/setup.cfg b/setup.cfg index 9ad0981..49c87ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,38 @@ [metadata] description-file = README.rst license_file = LICENSE.txt + + +[flake8] +max-line-length = 140 +exclude = */migrations/* + +[tool:pytest] +testpaths = tests +norecursedirs = + migrations + +python_files = + test_*.py + *_test.py + tests.py +addopts = + -ra + --strict + --doctest-modules + --doctest-glob=\*.rst + +python_versions = + py35 + py36 + py37 + +dependencies = +# 1.4: Django==1.4.16 !python_versions[py3*] +# 1.5: Django==1.5.11 +# 1.6: Django==1.6.8 +# 1.7: Django==1.7.1 !python_versions[py26] +# Deps commented above are provided as examples. That's what you would use in a Django project. + +environment_variables = + - diff --git a/setup.py b/setup.py index 329eb52..2adace7 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,10 @@ import os +from glob import glob +from os.path import basename +from os.path import splitext from setuptools import setup, find_packages - my_dir = os.path.dirname(os.path.realpath(__file__)) def readme(): @@ -15,7 +17,7 @@ def readme(): version='1.0.8a', description='A CLI application intending to automate offline music library building', long_description=readme(), - keywords='music album automation youtube audio metadata download', + keywords=['music album', 'automation', 'youtube', 'audio metadata', 'download'], classifiers=[ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', @@ -26,14 +28,15 @@ def readme(): 'Topic :: Multimedia :: Sound/Audio :: Conversion', 'Topic :: Multimedia :: Sound/Audio :: Editors', 'Intended Audience :: End Users/Desktop', - 'Intended Audience :: Science/Research', - ], + 'Intended Audience :: Science/Research' + ], url='https://github.com/boromir674/music-album-creator', author='Konstantinos Lampridis', author_email='k.lampridis@hotmail.com', license='GNU GPLv3', packages=find_packages(where='src'), - package_dir={'':'src'}, + package_dir={'': 'src'}, + py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], install_requires=['tqdm', 'click', 'sklearn', 'mutagen', 'PyInquirer', 'youtube_dl'], include_package_data=True, entry_points={ @@ -41,8 +44,9 @@ def readme(): 'create-album = music_album_creation.create_album:main', ] }, - setup_requires=['pytest-runner>=2.0',], - tests_require=['pytest',], + # TODO check if/where to put pytest + # setup_requires=['numpy>=1.11.0'], + tests_require=['tox', 'pytest'], # test_suite='', zip_safe=False ) diff --git a/src/music_album_creation/__init__.py b/src/music_album_creation/__init__.py index c545e69..f137543 100644 --- a/src/music_album_creation/__init__.py +++ b/src/music_album_creation/__init__.py @@ -6,3 +6,5 @@ from .downloading import YoutubeDownloader from .album_segmentation import AudioSegmenter + +__all__ = ['StringParser', 'MetadataDealer', 'FormatClassifier', 'YoutubeDownloader', 'AudioSegmenter'] diff --git a/src/music_album_creation/album_segmentation.py b/src/music_album_creation/album_segmentation.py index 5673fbc..b50250d 100644 --- a/src/music_album_creation/album_segmentation.py +++ b/src/music_album_creation/album_segmentation.py @@ -1,10 +1,8 @@ #!/usr/bin/python3 import re -import sys import time import subprocess -from warnings import warn from music_album_creation.tracks_parsing import StringParser diff --git a/src/music_album_creation/create_album.py b/src/music_album_creation/create_album.py index 8329f96..08395dc 100644 --- a/src/music_album_creation/create_album.py +++ b/src/music_album_creation/create_album.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import re import os import sys import glob @@ -37,7 +36,7 @@ @click.option('--album_artist', help="If given, then value shall be used as the TPE2 tag: 'Band/orchestra/accompaniment'. In the music player 'clementine' it corresponds to the 'Album artist' column") @click.option('--url', '-u', help='the youtube video url') def main(tracks_info, track_name, track_number, artist, album_artist, url): - ## CONFIG of the 'app' ## + # CONFIG of the 'app' # directory = '/tmp/gav' if os.path.isdir(directory): shutil.rmtree(directory) @@ -62,7 +61,7 @@ def main(tracks_info, track_name, track_number, artist, album_artist, url): print(e, '\n') if update_and_retry_dialog()['update-youtube-dl']: print("About to execute '{}'".format(YoutubeDownloader.update_backend_command)) - ro = YoutubeDownloader.update_backend() + _ = YoutubeDownloader.update_backend() else: print("Exiting ..") sys.exit(1) @@ -99,7 +98,7 @@ def main(tracks_info, track_name, track_number, artist, album_artist, url): fc = FormatClassifier.load(os.path.join(this_dir, "format_classification/data/model.pickle")) predicted_label = fc.is_durations([_[1] for _ in tracks_data]) # print('Predicted class {}; 0: timestamp input, 1:duration input'.format(predicted_label)) - answer = track_information_type_dialog(prediction={1:'durations'}.get(int(predicted_label), 'timestamps')) + answer = track_information_type_dialog(prediction={1: 'durations'}.get(int(predicted_label), 'timestamps')) if answer.startswith('Durations'): tracks_data = StringParser.duration_data_to_timestamp_data(tracks_data) @@ -136,7 +135,7 @@ def pathCompleter(self, text, state): This is the tab completer for systems paths. Only tested on *nix systems """ - line = readline.get_line_buffer().split() + _ = readline.get_line_buffer().split() return [x for x in glob.glob(text + '*')][state] def createListCompleter(self, ll): @@ -148,14 +147,11 @@ def createListCompleter(self, ll): """ def listCompleter(text, state): line = readline.get_line_buffer() - if not line: - print('CC1', c) return [c + " " for c in ll][state] - else: - print('CC2', c) return [c + " " for c in ll if c.startswith(line)][state] + self.listCompleter = listCompleter @@ -164,4 +160,4 @@ def listCompleter(text, state): readline.set_completer_delims('\t') readline.parse_and_bind("tab: complete") readline.set_completer(completer.pathCompleter) - main() \ No newline at end of file + main() diff --git a/src/music_album_creation/dialogs.py b/src/music_album_creation/dialogs.py index eff6d8f..08e5a1c 100644 --- a/src/music_album_creation/dialogs.py +++ b/src/music_album_creation/dialogs.py @@ -1,7 +1,7 @@ import os import shutil -from PyInquirer import style_from_dict, Token, prompt, Separator, Validator, ValidationError +from PyInquirer import prompt, Validator, ValidationError # __all__ = ['store_album_dialog', 'interactive_metadata_dialogs'] @@ -38,7 +38,7 @@ def track_information_type_dialog(prediction=''): choices = ['Timestamps', 'Durations'] questions = [ { - 'type': 'list', ## navigate with arrows through choices + 'type': 'list', # navigate with arrows through choices 'name': 'how-to-input-tracks', # type of is the format you prefer to input for providing the necessary information to segment an album 'message': 'What does the expected "hh:mm:ss" input represent?', @@ -187,7 +187,7 @@ def set_metadata_panel(artist=artist, album=album, year=year): 'type': 'input', 'name': 'year', 'message': "'year' tag", - 'default': year, # trick to allow empty value + 'default': year, # trick to allow empty value 'validate': NumberValidator, # 'filter': lambda val: int(val) }, diff --git a/src/music_album_creation/downloading.py b/src/music_album_creation/downloading.py index 1be26e1..e34acc0 100644 --- a/src/music_album_creation/downloading.py +++ b/src/music_album_creation/downloading.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 +import abc import re import subprocess @@ -60,7 +61,7 @@ def create_from_stderr(stderror, video_url): #### EXCEPTIONS -import abc + class AbstractYoutubeDownloaderError(metaclass=abc.ABCMeta): def __init__(self, *args, **kwargs): super().__init__() @@ -78,6 +79,7 @@ def __init__(self, *args, **kwargs): class TokenParameterNotInVideoInfoError(Exception, AbstractYoutubeDownloaderError): """Token error""" reg = '"token" parameter not in video info for unknown reason' + def __init__(self, video_url, stderror): AbstractYoutubeDownloaderError.__init__(self, video_url, stderror) Exception.__init__(self, self._msg) @@ -85,6 +87,7 @@ def __init__(self, video_url, stderror): class InvalidUrlError(Exception, AbstractYoutubeDownloaderError): """Invalid url error""" reg = r'is not a valid URL\.' + def __init__(self, video_url, stderror): AbstractYoutubeDownloaderError.__init__(self, video_url, stderror, msg="Invalid url '{}'.".format(video_url)) Exception.__init__(self, self._short_msg) @@ -92,6 +95,7 @@ def __init__(self, video_url, stderror): class UnavailableVideoError(Exception, AbstractYoutubeDownloaderError): """Wrong url error""" reg = r'ERROR: This video is unavailable\.' + def __init__(self, video_url, stderror): AbstractYoutubeDownloaderError.__init__(self, video_url, stderror, msg="Unavailable video at '{}'.".format(video_url)) Exception.__init__(self, self._msg) diff --git a/src/music_album_creation/format_classification/__init__.py b/src/music_album_creation/format_classification/__init__.py index 441c46b..28a3597 100644 --- a/src/music_album_creation/format_classification/__init__.py +++ b/src/music_album_creation/format_classification/__init__.py @@ -1,6 +1,8 @@ import os from .dataset import DatasetHandler -dataset_handler = DatasetHandler(datasets_root_dir=os.path.join(os.path.dirname(os.path.realpath(__file__)), "data")) +dataset_handler = DatasetHandler(datasets_root_dir=os.path.join(os.path.dirname(os.path.realpath(__file__)), "data")) from .tracks_format_classifier import FormatClassifier + +__all__ = ['FormatClassifier', 'dataset_handler'] diff --git a/src/music_album_creation/format_classification/dataset.py b/src/music_album_creation/format_classification/dataset.py index a9521f0..3e72eef 100644 --- a/src/music_album_creation/format_classification/dataset.py +++ b/src/music_album_creation/format_classification/dataset.py @@ -5,15 +5,13 @@ from warnings import warn import mutagen -# from librosa.core import get_duration +from tqdm import tqdm from music_album_creation.tracks_parsing import StringParser sp = StringParser.get_instance() -from tqdm import tqdm - class DatasetHandler: __instance = None @@ -36,7 +34,7 @@ def get_instance(cls, datasets_root_dir=''): def create_split(self, split, album_dirs, progress_bar=False): if progress_bar: - gen = tqdm(new_gen(album_dirs), total=len(album_dirs)*2, unit='datapoint') + gen = tqdm(new_gen(album_dirs), total=len(album_dirs) * 2, unit='datapoint') else: gen = new_gen(album_dirs) with open(os.path.join(self.datasets_root_dir, '{}{}'.format(split, self.post_fix)), 'w') as f: @@ -69,25 +67,25 @@ def create_datapoints(album_dirs_list, nb_datapoints=None, progress_bar=False, c :param class_ratio: :return: """ - d = [] - l = [] + feature_vectors = [] + class_labels = [] i = 0 assert nb_datapoints is None or type(nb_datapoints) == int if progress_bar: if not nb_datapoints: - gen = tqdm(new_gen(album_dirs_list), total=len(album_dirs_list)*2, unit='datapoint') + gen = tqdm(new_gen(album_dirs_list), total=len(album_dirs_list) * 2, unit='datapoint') else: gen = tqdm(new_gen(album_dirs_list), total=nb_datapoints, unit='datapoint') else: gen = new_gen(album_dirs_list) for i, datapoint in enumerate(gen): - d.append(datapoint[0]) # feature vector - l.append(datapoint[1]) # class label + feature_vectors.append(datapoint[0]) # feature vector + class_labels.append(datapoint[1]) # class label if nb_datapoints is not None and i == nb_datapoints - 1: break if nb_datapoints and i < nb_datapoints - 1: - warn("Requested {} datapoints but the {} albums available produced {}".format(nb_datapoints, len(album_dirs_list), len(d))) - return d, l + warn("Requested {} datapoints but the {} albums available produced {}".format(nb_datapoints, len(album_dirs_list), len(feature_vectors))) + return feature_vectors, class_labels def load_dataset_split(self, split): return self.load_dataset(os.path.join(self.datasets_root_dir, '{}{}'.format(split, self.post_fix))) @@ -106,12 +104,12 @@ def load_dataset(file_path): # return zip(*map(lambda x: (x.split(' ')[:-1], x.split(' ')[-1]), rows)) # return zip(*map(lambda x: (x[:-1], x[-1]), [map(int, r.split(' ')) for r in rows])) # [map(int, r.split(' ')) for r in rows] - #return zip(*list([(_.split(' ')[:-1], _.split(' ')[-1]) for _ in rows])) + # return zip(*list([(_.split(' ')[:-1], _.split(' ')[-1]) for _ in rows])) @staticmethod def save_dataset(file_path, feature_vectors, class_labels): with open(file_path, 'w') as f: - f.write('\n'.join('{} {}'.format(' '.join(str(el) for el in v), str(c)) for v,c in zip(feature_vectors, class_labels)) + '\n') + f.write('\n'.join('{} {}'.format(' '.join(str(el) for el in v), str(c)) for v, c in zip(feature_vectors, class_labels)) + '\n') def scan_for_albums(music_library, random=False): diff --git a/src/music_album_creation/format_classification/tracks_format_classifier.py b/src/music_album_creation/format_classification/tracks_format_classifier.py index 5daecdd..304bcd0 100644 --- a/src/music_album_creation/format_classification/tracks_format_classifier.py +++ b/src/music_album_creation/format_classification/tracks_format_classifier.py @@ -58,4 +58,4 @@ def predict(self, X): return self._estimator.predict(X) def score(self, X, y, sample_weight=None): - return self._estimator.score(X, y, sample_weight=sample_weight) \ No newline at end of file + return self._estimator.score(X, y, sample_weight=sample_weight) diff --git a/src/music_album_creation/metadata.py b/src/music_album_creation/metadata.py index f98a093..cc7f188 100644 --- a/src/music_album_creation/metadata.py +++ b/src/music_album_creation/metadata.py @@ -2,9 +2,6 @@ import re import glob import click -from functools import reduce -import mutagen -from mutagen import mp3 from mutagen.id3 import ID3, TPE1, TPE2, TRCK, TIT2, TALB, TDRC from collections import defaultdict @@ -17,14 +14,14 @@ class MetadataDealerType(type): def __parse_year(year): if year == '': return '' - c = re.match('0*(\d+)', year) + c = re.match(r'0*(\d+)', year) if not c: raise InvalidInputYearError("Input year tag '{}' is invalid".format(year)) - return re.match('0*(\d+)', year).group(1) + return re.match(r'0*(\d+)', year).group(1) def __new__(mcs, name, bases, attributes): x = super().__new__(mcs, name, bases, attributes) - x._filters = defaultdict(lambda : lambda y: y, track_number=lambda y: mcs.__parse_year(y)) + x._filters = defaultdict(lambda: lambda y: y, track_number=lambda y: mcs.__parse_year(y)) return x @@ -34,9 +31,9 @@ class MetadataDealer(metaclass=MetadataDealerType): # simply add keys and constructor pairs to enrich the support of the API for writting tags/frames to audio files # you can use the cls._filters to add a new post processing filter as shown in MetadataDealerType constructor above _d = {'artist': TPE1, # 4.2.1 TPE1 [#TPE1 Lead performer(s)/Soloist(s)] ; taken from http://id3.org/id3v2.3.0 - # in clementine temrs, it affects the 'Artist' tab but not the 'Album artist' + # in clementine temrs, it affects the 'Artist' tab but not the 'Album artist' 'album_artist': TPE2, # 4.2.1 TPE2 [#TPE2 Band/orchestra/accompaniment] - # in clementine terms, it affects the 'Artist' tab but not the 'Album artist' + # in clementine terms, it affects the 'Artist' tab but not the 'Album artist' 'album': TALB, # 4.2.1 TALB [#TALB Album/Movie/Show title] 'year': TDRC # TDRC (recording time) consolidates TDAT (date), TIME (time), TRDA (recording dates), and TYER (year). @@ -68,7 +65,7 @@ def write_metadata(cls, file, verbose=False, **kwargs): raise RuntimeError("Some of the input keys [{}] used to request the addition of metadata, do not correspoond" " to a tag/frame of the supported [{}]".format(', '.join(kwargs.keys()), ' '.join(cls._d))) audio = ID3(file) - for k,v in kwargs.items(): + for k, v in kwargs.items(): if bool(v): audio.add(cls._all[k](encoding=3, text=u'{}'.format(cls._filters[k](v)))) if verbose: @@ -89,8 +86,8 @@ class InvalidInputYearError(Exception): pass @click.command() @click.option('--album-dir', required=True, help="The directory where a music album resides. Currently only mp3 " - "files are supported as contents of the directory. Namely only " - "such files will be apprehended as tracks of the album.") + "files are supported as contents of the directory. Namely only " + "such files will be apprehended as tracks of the album.") @click.option('--track_name/--no-track_name', default=True, show_default=True, help='Whether to extract the track names from the mp3 files and write them as metadata correspondingly') @click.option('--track_number/--no-track_number', default=True, show_default=True, help='Whether to extract the track numbers from the mp3 files and write them as metadata correspondingly') @click.option('--artist', '-a', help="If given, then value shall be used as the TPE1 tag: 'Lead performer(s)/Soloist(s)'. In the music player 'clementine' it corresponds to the 'artist' column") diff --git a/src/music_album_creation/tracks_parsing.py b/src/music_album_creation/tracks_parsing.py index f5f5c8e..cd11ec5 100644 --- a/src/music_album_creation/tracks_parsing.py +++ b/src/music_album_creation/tracks_parsing.py @@ -93,7 +93,7 @@ def hhmmss_durations_to_timestamps(cls, hhmmss_list): @classmethod def _generate_timestamps(cls, hhmmss_list): - i, p = 1, '0:00' + p = '0:00' yield p for el in hhmmss_list[:-1]: _ = cls.add(p, el) @@ -148,7 +148,9 @@ def parse_album_info(video_title): year = r'\(?(\d{4})\)?' art = r'([\w ]*\w)' alb = r'([\w ]*\w)' - _reg = lambda x: re.compile(str('{}' * len(x)).format(*x)) + + def _reg(x): + return re.compile(str('{}' * len(x)).format(*x)) reg1 = _reg([art, sep1, alb, sep2, year]) m1 = reg1.search(video_title) @@ -223,6 +225,7 @@ def __track_file(cls, track_name): class Timestamp: instances = {} + def __new__(cls, *args, **kwargs): hhmmss = args[0] if hhmmss in cls.instances: @@ -245,28 +248,37 @@ def __init__(self, hhmmss): @staticmethod def from_duration(seconds): return Timestamp(time.strftime('%H:%M:%S', time.gmtime(seconds))) + def __repr__(self): return self._b + def __str__(self): return self._b + def __eq__(self, other): return str(self) == str(other) + def __int__(self): return self._s + def __lt__(self, other): return int(self) < int(other) + def __le__(self, other): return int(self) <= int(other) + def __gt__(self, other): return int(other) < int(self) + def __ge__(self, other): return int(other) <= int(self) + def __add__(self, other): return Timestamp.from_duration(int(self) + int(other)) + def __sub__(self, other): return Timestamp.from_duration(int(self) - int(other)) class WrongTimestampFormat(Exception): pass class TrackTimestampsSequenceError(Exception): pass - diff --git a/tests/test_classifier.py b/tests/test_classifier.py index 873c97f..cbbcfe7 100644 --- a/tests/test_classifier.py +++ b/tests/test_classifier.py @@ -1,8 +1,5 @@ -import os import pytest -import music_album_creation - from music_album_creation.format_classification import dataset_handler, FormatClassifier diff --git a/tests/test_create_album_program.py b/tests/test_create_album_program.py index 6e0c5c9..4be1101 100644 --- a/tests/test_create_album_program.py +++ b/tests/test_create_album_program.py @@ -1,12 +1,11 @@ import subprocess -import pytest from click.testing import CliRunner -import music_album_creation from music_album_creation.create_album import main + class TestCreateAlbum: def test_launching(self): diff --git a/tests/test_downloading.py b/tests/test_downloading.py index afaeb5f..d452ef0 100644 --- a/tests/test_downloading.py +++ b/tests/test_downloading.py @@ -1,7 +1,6 @@ import os import pytest -import music_album_creation from music_album_creation.downloading import YoutubeDownloader, InvalidUrlError, UnavailableVideoError @@ -27,4 +26,4 @@ def test_downloading_invalid_url(self, youtube): @pytest.mark.parametrize("url, target_file", [('https://www.youtube.com/watch?v=Q3dvbM6Pias', 'Rage Against The Machine - Testify')]) def test_downloading_valid_url(self, url, target_file, youtube): youtube.download(url, '/tmp', spawn=False, verbose=False, supress_stdout=True) - assert os.path.isfile('/tmp/'+target_file+'.mp3') \ No newline at end of file + assert os.path.isfile('/tmp/' + target_file + '.mp3') diff --git a/tests/test_segmenting.py b/tests/test_segmenting.py index d2eee17..21b3f09 100644 --- a/tests/test_segmenting.py +++ b/tests/test_segmenting.py @@ -2,17 +2,18 @@ import pytest import mutagen -import music_album_creation from music_album_creation.album_segmentation import AudioSegmenter, NotStartingFromZeroTimestampError from music_album_creation.tracks_parsing import WrongTimestampFormat, TrackTimestampsSequenceError this_dir = os.path.dirname(os.path.realpath(__file__)) + @pytest.fixture(scope='module') def test_audio_file_path(): return os.path.join(this_dir, 'know_your_enemy.mp3') + segmenter = AudioSegmenter() @@ -36,9 +37,9 @@ def test_wrong_timestamp_input(self, tmpdir, test_audio_file_path): segmenter.segment_from_list(test_audio_file_path, [['t1', '0:00'], ['t2', '1:a0'], ['t3', '1:35']], supress_stdout=True, verbose=False, sleep_seconds=0) @pytest.mark.parametrize("tracks, names, durations", [ - ("1. tr1 - 0:00\n2. tr2 - 1:12\n3. tr3 - 2:00\n", ['01 - tr1.mp3','02 - tr2.mp3','03 - tr3.mp3'], [72, 48, 236]), + ("1. tr1 - 0:00\n2. tr2 - 1:12\n3. tr3 - 2:00\n", ['01 - tr1.mp3', '02 - tr2.mp3', '03 - tr3.mp3'], [72, 48, 236]), pytest.param("1. tr1 - 0:00\n2. tr2 - 1:12\n3. tr3 - 1:00\n", - ['01 - tr1.mp3','02 - tr2.mp3','03 - tr3.mp3'], + ['01 - tr1.mp3', '02 - tr2.mp3', '03 - tr3.mp3'], [72, 48, 236], marks=pytest.mark.xfail), pytest.param("1. tr1 - 0:00\n2. tr2 - 1:72\n3. tr3 - 3:00\n", ['01 - tr1.mp3', '02 - tr2.mp3', '03 - tr3.mp3'], diff --git a/tests/test_splitters.py b/tests/test_splitters.py index 5ef3744..328986b 100644 --- a/tests/test_splitters.py +++ b/tests/test_splitters.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import pytest -import music_album_creation from music_album_creation.tracks_parsing import StringParser @@ -36,4 +35,4 @@ def test_tracks_string(self, tracks_string): ['Melancholy Sadie', '11:30'], ['Bowie’s Last Breath', '16:19'], ["I’m Not A Real Indian (But I Play One On TV)", '20:20'], - ['I Make Weird Choices', '23:44']] \ No newline at end of file + ['I Make Weird Choices', '23:44']] diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..696aed7 --- /dev/null +++ b/tox.ini @@ -0,0 +1,165 @@ +[tox] +envlist = + clean, + quality, + py35, + py36, + py37, +; reporting +; report +; coveralls + + +skip_missing_interpreters = {env:TOX_SKIP_MISSING_INTERPRETERS:True} + +[testenv] +basepython = + {docs,spell}: {env:TOXPYTHON:python3.6} + {bootstrap,clean,check,report,coveralls}: {env:TOXPYTHON:python3} +setenv = + PYTHONPATH={toxinidir}/tests + PYTHONUNBUFFERED=yes + # klein project below 2 + PIP_DISABLE_PIP_VERSION_CHECK=1 + VIRTUALENV_NO_DOWNLOAD=1 +passenv = + * + # See https://github.com/codecov/codecov-python/blob/5b9d539a6a09bc84501b381b563956295478651a/README.md#using-tox + codecov: TOXENV + codecov: CI + codecov: TRAVIS TRAVIS_* +deps = + pytest + # pytest-travis-fold + pytest-cov + +commands = + {posargs:pytest --cov --cov-report=term-missing -vv --ignore=src} + +; +;[testenv:check] +;deps = +; docutils +; check-manifest +; flake8 +; readme-renderer +; pygments +;skip_install = true +;commands = +; python setup.py check --strict --metadata --restructuredtext +; check-manifest {toxinidir} +; flake8 src tests setup.py +;; isort --verbose --check-only --diff --recursive src tests setup.py + + +[testenv:quality] +basepython = {env:TOXPYTHON:python3.6} +deps = + docutils + check-manifest + coverage + flake8 + readme-renderer + pygments +skip_install = true +commands = + python setup.py check --strict --metadata --restructuredtext + check-manifest {toxinidir} + flake8 src tests setup.py + + +[flake8] +# select the type of style errors to check +select = B,C,E,F,I,N,S,W +# disable skipping warning when '# noqa' is found +disable-noqa = True +# show the source file generating a warning +show-source = True +# check syntax of the doctests +doctests = True + +# Codes: http://flake8.pycqa.org/en/latest/user/error-codes.html +ignore = + # multiple spaces before operator + E221, + # too many blank lines + E302, + # too many blank lines + E303, + # expected 2 blank lines after class or function definition + E305, + # function name should be lowercase + N802, + # argument name should be lowercase + N803, + # first argument of a method should be named 'self' + N805, + # variable in function should be lowercase + N806, + # lowercase imported as non lowercase + N812, + # variable 'rawHeaders' in class scope should not be mixedCase + N815, + # variable 'noneIO' in global scope should not be mixedCase + N816, + # line break after binary operator (W503 and W504 are opposites) + W504, + # line too long + E501, + # class GavError(Exception): pass + E701, + # too many leading # for block comment + E266, + # missing whitespace around arithmetic operator + E226, + # module level import not at top of file + E402 + + +[testenv:clean] +deps = coverage +skip_install = true +commands = + ls -la + coverage erase + ls -la + +[testenv:reporting] +basepython = {env:TOXPYTHON:python3.6} +deps = + coverage + coveralls +skip_install = true +commands = + ls -la + coverage report + coverage html + coveralls + +;[testenv:report] +;deps = coverage +;skip_install = true +;commands = +; coverage report +; coverage html +; +;[testenv:coveralls] +;deps = +; coveralls +;skip_install = true +;commands = +;; coverage run --source=music_album_creation setup.py test +;; coveralls +; coveralls [] +;; +;;[travis:after] +;;envlist = report,coveralls + +[testenv:py35] +basepython = {env:TOXPYTHON:python3.5} + +[testenv:py36] +basepython = {env:TOXPYTHON:python3.6} + +[testenv:py37] +basepython = {env:TOXPYTHON:python3.7} From 5bfad6090ea2508fe67c4f46b238f991a9435d61 Mon Sep 17 00:00:00 2001 From: Konstantinos <> Date: Fri, 12 Jul 2019 03:28:02 +0200 Subject: [PATCH 5/9] Add exception for when youtube has too many requests. Adjust unittests to insist with delayed requests to ensure passing of test. Add an exception for ssl certificate verification error that can happen on the scrutinizer.io server when trying to download youtube and adjust unittests rather clumslyli but agilely. Configure scrutinizer.io server to perform Tests. Enable code-rating and code-duplication inspections, py-scrutinizer, pylint, build-test for python3.7 by running tox and attempt to perform coverage with combine command. NOTE: should try the append-flag for th epytest-cov command of tox and instead of using the scrutinizer feature for 'coverage combine' use the tox feature of running in the final env of the pipeline a 'coverage combine' command. Currently if a tox env requires a python interpreter that is not found in scrutinizer then the tox env test is skipped. Can be achived by manually installing the other interpreters on worker. Attemp to exclude 'build' directory so that scrutinizer does not look there and find duplicates with the source code. Reconfigure tox so that it executes coverage environemnt at the end of the pipeline. NOTE: should confirm the previous. Pass all travis and scrutinizer build tests. Fix README Fix exceptions WIP WIP Add default .scrutenizer.yml Include .scrutinizer.yml in the distribution Install tox in node 'tests' as dependency Change syntax Add an exception for ssl certificate verification error that can happen on the scrutinizer.io server when trying to download youtube video Fix regex of exception Add information to exception fired to check string not matching Wrap handling the ssl exception arround the test_valid_url unittest Wrap handling the ssl exception arround the test_valid_url unittest: Version 2 Wrap handling the ssl exception arround the test_valid_url unittest: Version 3 Update tox execution pipeline to end with 'report' and 'coveralls' envs Add more information to ExceptionFactory when not a predefined esception is fired Attempt1 Attempt2 Add flag for YoutubeDownloader to control suppressing certificate verification Fix logic of unittests Refactor test_downloading_false_youtube_url Add CertificateVerificationError class to the YoutubeDownloaderExceptionFactory Remove coveralls environemnt from tox pipeline. Firstly, because COVERALLS_REPO_TOKEN is not known to the scrutinizer worker and secondly because travis jobs send data to coveralls.io anyway. Enable coverage invocation in scrutinizer server and inlucde the .coverage file in distribution and version control Configure scrutinizer to perform coverage Configure scrutinizer to perform coverage Configure scrutinizer to perform coverage Configure scrutinizer to perform coverage Configure scrutinizer to perform coverage Configure scrutinizer to perform coverage Configure scrutinizer to perform coverage Configure scrutinizer to perform coverage Configure scrutinizer to perform coverage Another syntax for coverage Coverage automation attemp 1 Coverage automation attemp 2 Attemp to exclude 'build' direcory so that scrutinizer does not look there and find duplicates with the source code Delete .coverage file --- .scrutinizer.yml | 36 ++++++++++++++++ MANIFEST.in | 2 + README.rst | 10 +++++ src/music_album_creation/downloading.py | 56 ++++++++++++++++++++----- src/music_album_creation/metadata.py | 38 +++++++++-------- tests/test_downloading.py | 45 ++++++++++++++++++-- tox.ini | 50 +++++++--------------- 7 files changed, 170 insertions(+), 67 deletions(-) create mode 100644 .scrutinizer.yml diff --git a/.scrutinizer.yml b/.scrutinizer.yml new file mode 100644 index 0000000..82b0de8 --- /dev/null +++ b/.scrutinizer.yml @@ -0,0 +1,36 @@ +checks: + python: + code_rating: true + duplicate_code: true +build: + nodes: + analysis: + project_setup: + override: + - 'true' + tests: + override: + - py-scrutinizer-run + - + command: pylint-run + use_website_config: true + tests: + dependencies: + before: + - pip install tox + tests: + before: + - pip install coverage + after: + - + command: coverage combine + coverage: + file: .coverage + config_file: '.coveragerc' + format: py-cc +filter: + excluded_paths: + - '*/test/*' + - '*/build/*' + dependency_paths: + - 'lib/*' diff --git a/MANIFEST.in b/MANIFEST.in index a271bea..84b3796 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -13,7 +13,9 @@ graft src graft tests include .travis.yml +include .scrutinizer.yml include .coveragerc +include .coverage include tox.ini graft .travis diff --git a/README.rst b/README.rst index c6b132a..c4b4424 100644 --- a/README.rst +++ b/README.rst @@ -21,6 +21,7 @@ Overview * - package - | |version| |wheel| + .. |docs| image:: https://readthedocs.org/projects/music-album-creator/badge/?style=flat :target: https://readthedocs.org/projects/music-album-creator :alt: Documentation Status @@ -67,6 +68,15 @@ Installation pip install music-album-creator + +Usage +============ + +To run, simply execute:: + + create-album + + Documentation ============= diff --git a/src/music_album_creation/downloading.py b/src/music_album_creation/downloading.py index e34acc0..2d52731 100644 --- a/src/music_album_creation/downloading.py +++ b/src/music_album_creation/downloading.py @@ -16,7 +16,7 @@ class YoutubeDownloader: update_backend_command = ' '.join(update_command_args) @classmethod - def download(cls, video_url, directory, spawn=True, verbose=True, supress_stdout=False): + def download(cls, video_url, directory, spawn=True, verbose=True, supress_stdout=False, suppress_certificate_validation=False): """ Downloads a video from youtube given a url, converts it to mp3 and stores in input directory.\n :param str video_url: @@ -24,15 +24,23 @@ def download(cls, video_url, directory, spawn=True, verbose=True, supress_stdout :param bool spawn: :param bool verbose: :param bool supress_stdout: + :param bool suppress_certificate_validation: :return: """ + cls.__download(video_url, directory, spawn=spawn, verbose=verbose, supress_stdout=supress_stdout, suppress_certificate_validation=suppress_certificate_validation) + + @classmethod + def __download(cls, video_url, directory, **kwargs): args = cls._args[:-1] + ['{}/{}'.format(directory, cls._args[-1])] + [video_url] - if verbose: + # If suppress HTTPS certificate validation + if kwargs.get('suppress_certificate_validation', False): + args.insert(1, '--no-check-certificate') + if kwargs.get('verbose', False): print("Downloading stream '{}' and converting to mp3 ..".format(video_url)) - if spawn: - _ = subprocess.Popen(args, stderr=subprocess.STDOUT, **cls._capture_stdout[supress_stdout]) + if kwargs.get('spawn', True): + _ = subprocess.Popen(args, stderr=subprocess.STDOUT, **cls._capture_stdout[kwargs.get('supress_stdout', True)]) else: - ro = subprocess.run(args, stderr=subprocess.PIPE, **cls._capture_stdout[supress_stdout]) + ro = subprocess.run(args, stderr=subprocess.PIPE, **cls._capture_stdout[kwargs.get('supress_stdout', True)]) if ro.returncode != 0: stderr = str(ro.stderr, encoding='utf-8') raise YoutubeDownloaderErrorFactory.create_from_stderr(stderr, video_url) @@ -54,10 +62,12 @@ def create_with_message(msg): @staticmethod def create_from_stderr(stderror, video_url): - for subclass in (TokenParameterNotInVideoInfoError, InvalidUrlError, UnavailableVideoError): - if re.search(subclass.reg, stderror): + exception_classes = (TokenParameterNotInVideoInfoError, InvalidUrlError, UnavailableVideoError, TooManyRequestsError, CertificateVerificationError) + for subclass in exception_classes: + if subclass.reg.search(stderror): return subclass(video_url, stderror) - return Exception(AbstractYoutubeDownloaderError(video_url, stderror)._msg) + s = "NOTE: None of the predesinged exceptions' regexs [{}] matched. Perhaps you want to derive a new subclass from AbstractYoutubeDownloaderError to account for this youtube-dl exception with string to parse {}'".format(', '.join(['"{}"'.format(_.reg) for _ in exception_classes]), stderror) + return Exception(AbstractYoutubeDownloaderError(video_url, stderror)._msg + '\n' + s) #### EXCEPTIONS @@ -78,7 +88,7 @@ def __init__(self, *args, **kwargs): class TokenParameterNotInVideoInfoError(Exception, AbstractYoutubeDownloaderError): """Token error""" - reg = '"token" parameter not in video info for unknown reason' + reg = re.compile('"token" parameter not in video info for unknown reason') def __init__(self, video_url, stderror): AbstractYoutubeDownloaderError.__init__(self, video_url, stderror) @@ -86,7 +96,7 @@ def __init__(self, video_url, stderror): class InvalidUrlError(Exception, AbstractYoutubeDownloaderError): """Invalid url error""" - reg = r'is not a valid URL\.' + reg = re.compile(r'is not a valid URL\.') def __init__(self, video_url, stderror): AbstractYoutubeDownloaderError.__init__(self, video_url, stderror, msg="Invalid url '{}'.".format(video_url)) @@ -94,8 +104,32 @@ def __init__(self, video_url, stderror): class UnavailableVideoError(Exception, AbstractYoutubeDownloaderError): """Wrong url error""" - reg = r'ERROR: This video is unavailable\.' + reg = re.compile(r'ERROR: This video is unavailable\.') def __init__(self, video_url, stderror): AbstractYoutubeDownloaderError.__init__(self, video_url, stderror, msg="Unavailable video at '{}'.".format(video_url)) Exception.__init__(self, self._msg) + +class TooManyRequestsError(Exception, AbstractYoutubeDownloaderError): + """Too many requests (for youtube) to serve""" + reg = re.compile(r"(?:ERROR: Unable to download webpage: HTTP Error 429: Too Many Requests|WARNING: unable to download video info webpage: HTTP Error 429)") + + def __init__(self, video_url, stderror): + AbstractYoutubeDownloaderError.__init__(self, video_url, stderror, msg="Too many requests for youtube at the moment.".format(video_url)) + Exception.__init__(self, self._msg) + +class CertificateVerificationError(Exception, AbstractYoutubeDownloaderError): + """This can happen when downloading is requested from a server like scrutinizer.io\n + ERROR: Unable to download webpage: (caused by URLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] + certificate verify failed: unable to get local issuer certificate (_ssl.c:1056)'))) + """ + reg = re.compile(r"ERROR: Unable to download webpage: (caused by " + "URLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: " + "unable to get local issuer certificate (_ssl.c:1056)')))") + Exception.__init__(self, self._msg) diff --git a/src/music_album_creation/metadata.py b/src/music_album_creation/metadata.py index cc7f188..72acaa4 100644 --- a/src/music_album_creation/metadata.py +++ b/src/music_album_creation/metadata.py @@ -51,13 +51,14 @@ def set_album_metadata(self, album_directory, track_number=True, track_name=True self._write_metadata(album_directory, track_number=track_number, track_name=track_name, artist=artist, album_artist=album_artist, album=album, year=str(year), verbose=verbose) - def _write_metadata(self, album_directory, verbose=False, **kwargs): + @classmethod + def _write_metadata(cls, album_directory, verbose=False, **kwargs): files = glob.glob('{}/*.mp3'.format(album_directory)) if verbose: print('FILES\n', list(map(os.path.basename, files))) for file in files: - self.write_metadata(file, **dict(self._filter_auto_inferred(self._infer_track_number_n_name(file), **kwargs), - **{k: kwargs.get(k, '') for k in self._d.keys()})) + cls.write_metadata(file, **dict(cls._filter_auto_inferred(cls._infer_track_number_n_name(file), **kwargs), + **{k: kwargs.get(k, '') for k in cls._d.keys()})) @classmethod def write_metadata(cls, file, verbose=False, **kwargs): @@ -72,14 +73,18 @@ def write_metadata(cls, file, verbose=False, **kwargs): print("set '{}' with {}: {}={}".format(file, k, cls._all[k].__name__, cls._filters[k](v))) audio.save() - def _filter_auto_inferred(self, d, **kwargs): - for k in self._auto_data: + @classmethod + def _filter_auto_inferred(cls, d, **kwargs): + """Given a dictionary (like the one outputted by _infer_track_number_n_name), deletes entries unless it finds them declared in kwargs as key_name=True""" + for k in cls._auto_data: if not kwargs.get(k, False) and k in d: del d[k] return d - def _infer_track_number_n_name(self, file_name): - return {tt[0]: re.search(self.reg, file_name).group(i+1) for i, tt in enumerate(self._auto_data)} + @classmethod + def _infer_track_number_n_name(cls, file_name): + """Call this method to get a dict like {'track_number': 'number', 'track_name': 'name'} from input file name with format like '1. - Loyal to the Pack.mp3'; number must be included!""" + return {tt[0]: re.search(cls.reg, file_name).group(i+1) for i, tt in enumerate(cls._auto_data)} class InvalidInputYearError(Exception): pass @@ -88,19 +93,16 @@ class InvalidInputYearError(Exception): pass @click.option('--album-dir', required=True, help="The directory where a music album resides. Currently only mp3 " "files are supported as contents of the directory. Namely only " "such files will be apprehended as tracks of the album.") -@click.option('--track_name/--no-track_name', default=True, show_default=True, help='Whether to extract the track names from the mp3 files and write them as metadata correspondingly') -@click.option('--track_number/--no-track_number', default=True, show_default=True, help='Whether to extract the track numbers from the mp3 files and write them as metadata correspondingly') -@click.option('--artist', '-a', help="If given, then value shall be used as the TPE1 tag: 'Lead performer(s)/Soloist(s)'. In the music player 'clementine' it corresponds to the 'artist' column") -@click.option('--album_artist', help="If given, then value shall be used as the TPE2 tag: 'Band/orchestra/accompaniment'. In the music player 'clementine' it corresponds to the 'Album artist' column") -def main(album_dir, track_name, track_number, artist, album_artist): +@click.option('--track_name/--no-track_name', default=True, show_default=True, help='Whether to extract the track names from the mp3 files and write them as metadata correspondingly.') +@click.option('--track_number/--no-track_number', default=True, show_default=True, help='Whether to extract the track numbers from the mp3 files and write them as metadata correspondingly.') +@click.option('--artist', '-a', help="If given, then value shall be used as the TPE1 tag: 'Lead performer(s)/Soloist(s)'. In the music player 'clementine' it corresponds to the 'Artist' column.") +@click.option('--album_artist', '-aa', help="If given, then value shall be used as the TPE2 tag: 'Band/orchestra/accompaniment'. In the music player 'clementine' it corresponds to the 'Album artist' column.") +@click.option('--album', '-al', help="If given, then value shall be used as the TALB tag: 'Album/Movie/Show title'. In the music player 'clementine' it corresponds to the 'Album' column.") +@click.option('--year', 'y', help="If given, then value shall be used as the TDRC tag: 'Recoring time'. In the music player 'clementine' it corresponds to the 'Year' column.") +def main(album_dir, track_name, track_number, artist, album_artist, album, year): md = MetadataDealer() - md.set_album_metadata(album_dir, track_number=track_number, track_name=track_name, artist=artist, album_artist=album_artist, verbose=True) + md.set_album_metadata(album_dir, track_number=track_number, track_name=track_name, artist=artist, album_artist=album_artist, album=album, year=year, verbose=True) -def test(): - al = '/data/projects/music-album-creator/lttp' - md = MetadataDealer() - md.set_album_metadata(al, track_name=True, track_number=True, artist='gg', album_artist='navi', album='alb', year='2009', verbose=True) - if __name__ == '__main__': main() diff --git a/tests/test_downloading.py b/tests/test_downloading.py index d452ef0..5ab5585 100644 --- a/tests/test_downloading.py +++ b/tests/test_downloading.py @@ -1,7 +1,8 @@ import os +from time import sleep import pytest -from music_album_creation.downloading import YoutubeDownloader, InvalidUrlError, UnavailableVideoError +from music_album_creation.downloading import YoutubeDownloader, InvalidUrlError, UnavailableVideoError, TooManyRequestsError, CertificateVerificationError @pytest.fixture(scope='module') @@ -15,9 +16,32 @@ class TestYoutubeDownloader: duration = '3:43' duration_in_seconds = 223 + + def attemp_download(self, url, times=10, sleep_seconds=1): + i = 0 + while i < times: + try: + YoutubeDownloader.download(url, '/tmp/', spawn=False, verbose=False, supress_stdout=True) + break + except TooManyRequestsError as e: + print(e) + i = times + def test_downloading_false_youtube_url(self, youtube): + suppress_certificate_validation = False + i = 0 with pytest.raises(UnavailableVideoError): - youtube.download(self.NON_EXISTANT_YOUTUBE_URL, '/tmp/', spawn=False, verbose=False, supress_stdout=True) + while i < 10: + try: + youtube.download(self.NON_EXISTANT_YOUTUBE_URL, '/tmp/', spawn=False, verbose=False, supress_stdout=True, suppress_certificate_validation=suppress_certificate_validation) + except CertificateVerificationError as e: + print('Attempt {}: {}'.format(i + 1, e)) + suppress_certificate_validation = True + sleep(0.3) + except TooManyRequestsError as e: + print('Attempt {}: {}'.format(i+1, e)) + sleep(1) + i += 1 def test_downloading_invalid_url(self, youtube): with pytest.raises(InvalidUrlError): @@ -25,5 +49,18 @@ def test_downloading_invalid_url(self, youtube): @pytest.mark.parametrize("url, target_file", [('https://www.youtube.com/watch?v=Q3dvbM6Pias', 'Rage Against The Machine - Testify')]) def test_downloading_valid_url(self, url, target_file, youtube): - youtube.download(url, '/tmp', spawn=False, verbose=False, supress_stdout=True) - assert os.path.isfile('/tmp/' + target_file + '.mp3') + suppress_certificate_validation = False + i = 0 + while i < 10: + try: + youtube.download(url, '/tmp', spawn=False, verbose=False, supress_stdout=True, suppress_certificate_validation=suppress_certificate_validation) + assert os.path.isfile('/tmp/' + target_file + '.mp3') + break + except TooManyRequestsError as e: + print(e) + sleep(1) + except CertificateVerificationError as e: + suppress_certificate_validation = True + print(e) + sleep(0.3) + i += 1 diff --git a/tox.ini b/tox.ini index 696aed7..a0bfc9c 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,8 @@ envlist = py35, py36, py37, + report, +; coveralls ; reporting ; report ; coveralls @@ -36,21 +38,6 @@ deps = commands = {posargs:pytest --cov --cov-report=term-missing -vv --ignore=src} -; -;[testenv:check] -;deps = -; docutils -; check-manifest -; flake8 -; readme-renderer -; pygments -;skip_install = true -;commands = -; python setup.py check --strict --metadata --restructuredtext -; check-manifest {toxinidir} -; flake8 src tests setup.py -;; isort --verbose --check-only --diff --recursive src tests setup.py - [testenv:quality] basepython = {env:TOXPYTHON:python3.6} @@ -120,9 +107,7 @@ ignore = deps = coverage skip_install = true commands = - ls -la coverage erase - ls -la [testenv:reporting] basepython = {env:TOXPYTHON:python3.6} @@ -131,29 +116,26 @@ deps = coveralls skip_install = true commands = - ls -la coverage report coverage html coveralls -;[testenv:report] -;deps = coverage -;skip_install = true -;commands = -; coverage report -; coverage html -; -;[testenv:coveralls] -;deps = -; coveralls -;skip_install = true -;commands = +[testenv:report] +deps = coverage +skip_install = true +commands = + coverage report + coverage html + +[testenv:coveralls] +deps = + coveralls +skip_install = true +commands = + coveralls [] ;; coverage run --source=music_album_creation setup.py test ;; coveralls -; coveralls [] -;; -;;[travis:after] -;;envlist = report,coveralls + [testenv:py35] basepython = {env:TOXPYTHON:python3.5} From 0a43b450ba9a0349459e561713e982629a2a7983 Mon Sep 17 00:00:00 2001 From: Konstantinos <> Date: Sat, 13 Jul 2019 16:25:49 +0200 Subject: [PATCH 6/9] Update badges for dev branch --- README.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index c4b4424..5d16a6f 100644 --- a/README.rst +++ b/README.rst @@ -18,6 +18,8 @@ Overview * - tests - | |travis| | |coveralls| + | |scrutinizer_code_quality| + | |code_intelligence_status| * - package - | |version| |wheel| @@ -26,7 +28,7 @@ Overview :target: https://readthedocs.org/projects/music-album-creator :alt: Documentation Status -.. |travis| image:: https://travis-ci.org/boromir674/music-album-creator.svg?branch=master +.. |travis| image:: https://travis-ci.org/boromir674/music-album-creator.svg?branch=dev :alt: Travis-CI Build Status :target: https://travis-ci.org/boromir674/music-album-creator @@ -34,6 +36,14 @@ Overview :alt: Coverage Status :target: https://coveralls.io/github/boromir674/music-album-creator?branch=dev +.. |scrutinizer_code_quality| image:: https://scrutinizer-ci.com/g/boromir674/music-album-creator/badges/quality-score.png?b=dev + :alt: Code Quality + :target: https://scrutinizer-ci.com/g/boromir674/music-album-creator/?branch=dev + +.. |code_intelligence_status| image:: https://scrutinizer-ci.com/g/boromir674/music-album-creator/badges/code-intelligence.svg?b=dev + :alt: Code Intelligence + :target: https://scrutinizer-ci.com/code-intelligence + .. |version| image:: https://img.shields.io/pypi/v/music-album-creator.svg :alt: PyPI Package latest release :target: https://pypi.org/project/music-album-creator From 92d91628976d5ab165406bf96b906a26dd7d66c5 Mon Sep 17 00:00:00 2001 From: Konstantinos <> Date: Sun, 14 Jul 2019 13:27:28 +0200 Subject: [PATCH 7/9] Use a common set of regexs to parse 'track_number - track_name' kind of strings. Add unittest for parsing track names containing the '-' character. Implement a StringToDict reg parser Fix regexes Fix regexes WIP Finalize All tests Clean --- src/music_album_creation/tracks_parsing.py | 135 +++++++++++++-------- tests/test_splitters.py | 6 +- 2 files changed, 87 insertions(+), 54 deletions(-) diff --git a/src/music_album_creation/tracks_parsing.py b/src/music_album_creation/tracks_parsing.py index cd11ec5..d65a01d 100644 --- a/src/music_album_creation/tracks_parsing.py +++ b/src/music_album_creation/tracks_parsing.py @@ -3,10 +3,69 @@ import time +class StringToDictParser: + """Parses album information out of video title string""" + check = re.compile(r'^s([1-9]\d*)$') + + class AlbumInfoEntity: + def __init__(self, name, reg): + self.name = name + self.reg = reg + + def __str__(self): + return self.reg + + class RegexSequence: + def __init__(self, data): + self._keys = [d.name for d in data if hasattr(d, 'name')] + self._regex = r'{}'.format(''.join(str(d) for d in data)) + + def search_n_dict(self, string): + return dict(_ for _ in + zip(self._keys, list(getattr(re.search(self._regex, string), 'groups', lambda: ['', '', ''])())) + if _[1]) + + def __str__(self): + return self._regex + + def __init__(self, entities, separators): + assert all(type(x) == str for x in separators) + self.entities = {k: self.AlbumInfoEntity(k, v) for k, v in entities.items()} + self.separators = separators + + def __call__(self, *args, **kwargs): + title = args[0] + design = kwargs['design'] + assert all(0 <= len(x) <= len(self.entities) + len(self.separators) and all(type(y) == str for y in x) for x in design) + assert all(all(self.check.match(y) for y in x if y.startswith('s')) for x in design) + rregs = [self.RegexSequence([_ for _ in self._yield_reg_comp(d)]) for d in design] + return max([r.search_n_dict(title) for r in rregs], key=lambda x: len(x)) + + def _yield_reg_comp(self, kati): + for k in kati: + if k.startswith('s'): + yield self.separators[int(self.check.match(k).group(1)) - 1] + else: + yield self.entities[k] + + class StringParser: __instance = None - timestamp_objects = {} - sep = r'(?:[\t ]+|[\t ]*[\-\.]+[\t ]*)' + + track_number = r'\d{1,2}' + track_name = r'[\w\'\(\) \-’]*[\w)]' + sep = r'(?:[\t ]+|[\t ]*[\.\-,]+[\t ]*)' + extension = r'.mp3' + hhmmss = r'(?:\d?\d:)*\d?\d' + + ## to parse from youtube video title string + sep1 = r'[\t ]*[\-\.][\t ]*' + sep2 = r'[\t \-\.]+' + year = r'\(?(\d{4})\)?' + art = r'([\w ]*\w)' + alb = r'([\w ]*\w)' + + album_info_parser = StringToDictParser({'artist': art, 'album': alb, 'year': year}, [sep1, sep2]) def __new__(cls, *args, **kwargs): if not cls.__instance: @@ -35,7 +94,7 @@ def _gen_timestamp_data(duration_data): @classmethod def parse_hhmmss_string(cls, tracks): - """Call this method to transform a '\n' separabale string of album tracks to a list of lists. Inner lists contains [track_name, hhmmss_timestamp].\n + """Call this method to transform a '\n' separabale string of album tracks (eg copy-pasted from video description) to a list of lists. Inner lists contains [track_name, hhmmss_timestamp].\n :param str tracks: :return: """ @@ -47,7 +106,6 @@ def _parse_string(cls, tracks): :param str tracks: a '\n' separable string of lines coresponding to the tracks information :return: """ - # regex = re.compile('(?:\d{1,2}[ \t]*[\.\-,][ \t]*|[\t ]+)?([\w\'\(\) ]*[\w)])' + cls.sep + '((?:\d?\d:)*\d?\d)$') for i, line in enumerate(_.strip() for _ in tracks.split('\n')): if line == '': continue @@ -59,13 +117,11 @@ def _parse_string(cls, tracks): @classmethod def _parse_track_line(cls, track_line): """Parses a string line such as '01. Doteru 3:45'""" - regex = re.compile(r"""^(?:\d{1,2}(?:[\ \t]*[\.\-,][\ \t]*|[\t\ ]+))? # potential track number (eg 01) included is ignored - ([\w\'\(\) ’]*[\w)]) # track name - (?:[\t ]+|[\t ]*[\-\.]+[\t ]*) # separator between name and time - ((?:\d?\d:)*\d?\d)$ # time in hh:mm:ss format""", re.X) - # regex = re.compile('^(?:\d{1,2}([\ \t]*[\.\-,][ \t]*|[\t ]+))?([\w\'\(\) ]*[\w)])' + cls.sep + '((?:\d?\d:)*\d?\d)$') - # regex = re.compile( - # '^(?:\d{1,2}[ \t]*[\.\-,][ \t]*|[\t ]+)?([\w\'\(\) ]*[\w)])' + cls.sep + '((?:\d?\d:)*\d?\d)$') + # regex = re.compile(r"""^(?:\d{1,2}(?:[\ \t]*[\.\-,][\ \t]*|[\t\ ]+))? # potential track number (eg 01) included is ignored + # ([\w\'\(\) \-’]*[\w)]) # track name + # (?:[\t ]+|[\t ]*[\-\.]+[\t ]*) # separator between name and time + # ((?:\d?\d:)*\d?\d)$ # time in hh:mm:ss format""", re.X) + regex = re.compile(r"^(?:{}{})?({}){}({})$".format(cls.track_number, cls.sep, cls.track_name, cls.sep, cls.hhmmss)) return list(regex.search(track_line.strip()).groups()) @classmethod @@ -82,10 +138,7 @@ def parse_tracks_hhmmss(cls, tracks_row_strings): :return: a list of lists with each inner list corresponding to each input string row and having 2 elements: the track name and the timestamp :rtype: list """ - regex = re.compile(r'(?:\d{1,2}[ \t]*[\.\-,][ \t]*|[\t ]+)?([\w ]*\w)' + cls.sep + r'((?:\d?\d:)*\d?\d)') - # regex = re.compile('(?:\d{1,2}(?:[ \t]*[\.\-,][ \t]*|[\t ])+)?([\w ]*\w)' + cls.sep + '((?:\d?\d:)*\d\d)') - - return [list(_) for _ in regex.findall(tracks_row_strings)] + return cls.parse_hhmmss_string(tracks_row_strings) @classmethod def hhmmss_durations_to_timestamps(cls, hhmmss_list): @@ -102,7 +155,12 @@ def _generate_timestamps(cls, hhmmss_list): @classmethod def convert_to_timestamps(cls, tracks_row_strings): - """Accepts tracks durations in hh:mm:ss format; one per row""" + """Call this method to transform a '\n' separabale string of album tracks (eg copy-pasted from the youtube video description) that represents durations (in hhmmss format) + to a list of strings with each track's starting timestamp in hhmmss format.\n + :param str tracks_row_strings: + :return: the list of each track's timestamp + :rtype: list + """ lines = cls.parse_tracks_hhmmss(tracks_row_strings) # list of lists i = 1 timestamps = ['0:00'] @@ -131,46 +189,17 @@ def time_format(seconds): """Call this method to transform an integer representing time duration in seconds to its equivalent hh:mm:ss formatted string representeation""" return time.strftime('%H:%M:%S', time.gmtime(seconds)) - @staticmethod - def parse_album_info(video_title): - """Parses a video title string into 'artist', 'album' and 'year' fields.\n - Can parse patters: - - Artist Album Year\n - - Album Year\n - - Artist Album\n - - Album\n - :param video_title: + @classmethod + def parse_album_info(cls, video_title): + """Call to parse a video title string into a hash (dictionary) of potentially all the 3 fields; 'artist', 'album' and 'year'.\n + :param str video_title: :return: the exracted values as a dictionary having maximally keys: {'artist', 'album', 'year'} :rtype: dict """ - sep1 = r'[\t ]*[\-\.][\t ]*' - sep2 = r'[\t \-\.]+' - year = r'\(?(\d{4})\)?' - art = r'([\w ]*\w)' - alb = r'([\w ]*\w)' - - def _reg(x): - return re.compile(str('{}' * len(x)).format(*x)) - - reg1 = _reg([art, sep1, alb, sep2, year]) - m1 = reg1.search(video_title) - if m1: - return {'artist': m1.group(1), 'album': m1.group(2), 'year': m1.group(3)} - - m1 = _reg([alb, sep2, year]).search(video_title) - if m1: - return {'album': m1.group(1), 'year': m1.group(2)} - - reg2 = _reg([art, sep1, alb]) - m2 = reg2.search(video_title) - if m2: - return {'artist': m2.group(1), 'album': m2.group(2)} - - reg3 = _reg([alb]) - m3 = reg3.search(video_title) - if m3: - return {'album': m3.group(1)} - return {} + return cls.album_info_parser(video_title, design=[['artist', 's1', 'album', 's2', 'year'], + ['artist', 's1', 'album'], + ['album', 's2', 'year'], + ['album']]) @classmethod def convert_tracks_data(cls, data, album_file, target_directory=''): diff --git a/tests/test_splitters.py b/tests/test_splitters.py index 328986b..e9ad20c 100644 --- a/tests/test_splitters.py +++ b/tests/test_splitters.py @@ -10,7 +10,11 @@ class TestSplitters: ("1 A track - 0:00", "A track", "0:00"), ("01 A track - 0:00", "A track", "0:00"), ("01. A track - 0:00", "A track", "0:00"), - ("3. Uber en Colère - 9:45", "Uber en Colère", '9:45') + ("3. Uber en Colère - 9:45", "Uber en Colère", '9:45'), + ("3. Delta-v - 20:04", 'Delta-v', '20:04'), + ("3. Delta-v - 0:00", 'Delta-v', '0:00'), + ("3 Delta-v - 20:04", 'Delta-v', '20:04'), + ("3 Delta-v - 0:00", 'Delta-v', '0:00'), ]) def test_tracks_line_parsing(self, track_line, name, time): assert StringParser._parse_track_line(track_line) == [name, time] From 282ab0012a91089cbc44efda2f2674ebd2c1d4d4 Mon Sep 17 00:00:00 2001 From: Konstantinos <> Date: Sun, 14 Jul 2019 13:27:28 +0200 Subject: [PATCH 8/9] Add some rules to exclude matching code lines from checking for coverage. Put 'check' TOXENV in 1st stage of travis and allow the lint and readme valid-rst checks to fail. Fix regexes Fix regexes WIP Finalize All tests Clean Add some rules to exclude lines from checking for coverage Tox envs refactoring Put check TOXENV in 1st stage of travis and allow the lint and readme valid-rst checks to fail --- .coveragerc | 10 ++++ .travis.yml | 17 +++---- src/music_album_creation/tracks_parsing.py | 57 ++++++++++++---------- tox.ini | 48 ++++++++++++------ 4 files changed, 82 insertions(+), 50 deletions(-) diff --git a/.coveragerc b/.coveragerc index 19e5ee1..9a87a3b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -14,3 +14,13 @@ parallel = true show_missing = true precision = 2 omit = *migrations* +# Regexes for lines to exclude from consideration +exclude_lines = + except ImportError + raise NotImplementedError + pass + ABCmeta + abstractmethod + abstractproperty + abstractclassmethod + warnings.warn diff --git a/.travis.yml b/.travis.yml index 5cafbdd..3b7c594 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,29 +45,28 @@ script: .travis/run-tox.sh jobs: fail_fast: true include: -# - stage: clean coverage data -# python: '3.6' -# env: TOXENV=clean - - stage: run unittests + - stage: Check + env: TOXENV=clean + - stage: run tests python: '3.6' env: TOXENV=quality script: - export TOX_SKIP_MISSING_INTERPRETERS="False" - tox - - stage: run unittests + - stage: run tests python: '3.5' env: TOXENV=py35 - - stage: run unittests + - stage: run tests python: '3.6' env: TOXENV=py36 - - stage: run unittests + - stage: run tests python: '3.7' env: TOXENV=py37 # - stage: send coverage data to coveralls.io # python: '3.6' # env: TOXENV=reporting -# allow_failures: -# - env: TOXENV=reporting + allow_failures: + - env: TOXENV=quality after_failure: - more .tox/log/* | cat diff --git a/src/music_album_creation/tracks_parsing.py b/src/music_album_creation/tracks_parsing.py index d65a01d..98633ab 100644 --- a/src/music_album_creation/tracks_parsing.py +++ b/src/music_album_creation/tracks_parsing.py @@ -7,47 +7,46 @@ class StringToDictParser: """Parses album information out of video title string""" check = re.compile(r'^s([1-9]\d*)$') - class AlbumInfoEntity: - def __init__(self, name, reg): - self.name = name - self.reg = reg - - def __str__(self): - return self.reg - - class RegexSequence: - def __init__(self, data): - self._keys = [d.name for d in data if hasattr(d, 'name')] - self._regex = r'{}'.format(''.join(str(d) for d in data)) - - def search_n_dict(self, string): - return dict(_ for _ in - zip(self._keys, list(getattr(re.search(self._regex, string), 'groups', lambda: ['', '', ''])())) - if _[1]) - - def __str__(self): - return self._regex - def __init__(self, entities, separators): assert all(type(x) == str for x in separators) - self.entities = {k: self.AlbumInfoEntity(k, v) for k, v in entities.items()} + self.entities = {k: AlbumInfoEntity(k, v) for k, v in entities.items()} self.separators = separators def __call__(self, *args, **kwargs): title = args[0] design = kwargs['design'] assert all(0 <= len(x) <= len(self.entities) + len(self.separators) and all(type(y) == str for y in x) for x in design) - assert all(all(self.check.match(y) for y in x if y.startswith('s')) for x in design) - rregs = [self.RegexSequence([_ for _ in self._yield_reg_comp(d)]) for d in design] + assert all(all(StringToDictParser.check.match(y) for y in x if y.startswith('s')) for x in design) + rregs = [RegexSequence([_ for _ in self._yield_reg_comp(d)]) for d in design] return max([r.search_n_dict(title) for r in rregs], key=lambda x: len(x)) def _yield_reg_comp(self, kati): for k in kati: if k.startswith('s'): - yield self.separators[int(self.check.match(k).group(1)) - 1] + yield self.separators[int(StringToDictParser.check.match(k).group(1)) - 1] else: yield self.entities[k] +class AlbumInfoEntity: + def __init__(self, name, reg): + self.name = name + self.reg = reg + + def __str__(self): + return self.reg + + +class RegexSequence: + def __init__(self, data): + self._keys = [d.name for d in data if hasattr(d, 'name')] + self._regex = r'{}'.format(''.join(str(d) for d in data)) + + def search_n_dict(self, string): + return dict(_ for _ in zip(self._keys, list(getattr(re.search(self._regex, string), 'groups', lambda: ['', '', ''])())) if _[1]) + + def __str__(self): + return self._regex + class StringParser: __instance = None @@ -106,6 +105,7 @@ def _parse_string(cls, tracks): :param str tracks: a '\n' separable string of lines coresponding to the tracks information :return: """ + # regex = re.compile('(?:\d{1,2}[ \t]*[\.\-,][ \t]*|[\t ]+)?([\w\'\(\) ]*[\w)])' + cls.sep + '((?:\d?\d:)*\d?\d)$') for i, line in enumerate(_.strip() for _ in tracks.split('\n')): if line == '': continue @@ -191,7 +191,12 @@ def time_format(seconds): @classmethod def parse_album_info(cls, video_title): - """Call to parse a video title string into a hash (dictionary) of potentially all the 3 fields; 'artist', 'album' and 'year'.\n + """Call to parse a video title string into a hash (dictionary) of potentially all 'artist', 'album' and 'year' fields.\n + Can parse patters: + - Artist Album Year\n + - Artist Album\n + - Album Year\n + - Album\n :param str video_title: :return: the exracted values as a dictionary having maximally keys: {'artist', 'album', 'year'} :rtype: dict diff --git a/tox.ini b/tox.ini index a0bfc9c..469b31d 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ skip_missing_interpreters = {env:TOX_SKIP_MISSING_INTERPRETERS:True} [testenv] basepython = {docs,spell}: {env:TOXPYTHON:python3.6} - {bootstrap,clean,check,report,coveralls}: {env:TOXPYTHON:python3} + {bootstrap,clean,check,report,coveralls,quality}: {env:TOXPYTHON:python3} setenv = PYTHONPATH={toxinidir}/tests PYTHONUNBUFFERED=yes @@ -39,21 +39,29 @@ commands = {posargs:pytest --cov --cov-report=term-missing -vv --ignore=src} -[testenv:quality] -basepython = {env:TOXPYTHON:python3.6} +# This env must succeed in order to be meaningful to proceed running the rest of the environments +[testenv:clean] deps = - docutils - check-manifest coverage + check-manifest +skip_install = true +commands = + coverage erase + check-manifest {toxinidir} + python setup.py check --strict --metadata + + +# This env can be potentially allowed to fail +[testenv:quality] +deps = flake8 - readme-renderer pygments + docutils + readme-renderer skip_install = true commands = - python setup.py check --strict --metadata --restructuredtext - check-manifest {toxinidir} flake8 src tests setup.py - + python setup.py check --strict --restructuredtext [flake8] # select the type of style errors to check @@ -103,12 +111,6 @@ ignore = E402 -[testenv:clean] -deps = coverage -skip_install = true -commands = - coverage erase - [testenv:reporting] basepython = {env:TOXPYTHON:python3.6} deps = @@ -145,3 +147,19 @@ basepython = {env:TOXPYTHON:python3.6} [testenv:py37] basepython = {env:TOXPYTHON:python3.7} + + + +;[testenv:quality] +;basepython = {env:TOXPYTHON:python3.6} +;deps = +; docutils +; check-manifest +; coverage +; flake8 +; readme-renderer +; pygments +;skip_install = true +;commands = +; python setup.py check --strict --metadata --restructuredtext +; flake8 src tests setup.py From 6805f8fa9c906bd10dea93c5c71f9bf09ab541e7 Mon Sep 17 00:00:00 2001 From: Konstantinos <> Date: Tue, 16 Jul 2019 21:54:18 +0200 Subject: [PATCH 9/9] Run pytest-cov with append flag in tox, clean code Modify setup.cfg Try to make scrutinizer.io recognize the coverage data Make travis send coverage data on unittests build jobs WIP Comment out coveralls from tox pipeline --- .scrutinizer.yml | 3 +- .travis.yml | 105 +++------------------------- .travis/deploy.sh | 3 - .travis/install-tox.sh | 6 -- .travis/run-tox.sh | 10 --- setup.cfg | 45 +++++------- setup.py | 60 +++++++++++----- src/music_album_creation/dialogs.py | 16 ++--- tox.ini | 43 ++---------- 9 files changed, 90 insertions(+), 201 deletions(-) delete mode 100644 .travis/deploy.sh delete mode 100644 .travis/install-tox.sh delete mode 100644 .travis/run-tox.sh diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 82b0de8..aca88ba 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -21,7 +21,8 @@ build: tests: before: - pip install coverage - after: + override: + - tox - command: coverage combine coverage: diff --git a/.travis.yml b/.travis.yml index 3b7c594..557c198 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,58 +1,42 @@ +os: linux language: python + env: global: - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so - SEGFAULT_SIGNALS=all - -#matrix: -# include: -# - python: '3.6' -# env: -# - TOXENV=check -# - env: -# - TOXENV=py35 -# python: '3.5' -# - env: -# - TOXENV=py36 -# python: '3.6' -# - env: -# - TOXENV=py37 -# python: '3.7' - + - TOX_SKIP_MISSING_INTERPRETERS="False" before_install: - python --version -# - pip install coveralls - uname -a - lsb_release -a - - chmod +x .travis/install-tox.sh - - chmod +x .travis/run-tox.sh - sudo apt-get install ffmpeg -# - chmod +x .travis/deploy.sh + install: - - .travis/install-tox.sh -# - pip install tox-travis + - pip install tox + - pip install coveralls - virtualenv --version - easy_install --version - pip --version - tox --version - - pip install coveralls cache: pip -script: .travis/run-tox.sh +script: + - tox + - coveralls jobs: fail_fast: true include: - stage: Check env: TOXENV=clean + script: tox - stage: run tests python: '3.6' env: TOXENV=quality - script: - - export TOX_SKIP_MISSING_INTERPRETERS="False" - - tox + script: tox - stage: run tests python: '3.5' env: TOXENV=py35 @@ -62,9 +46,6 @@ jobs: - stage: run tests python: '3.7' env: TOXENV=py37 -# - stage: send coverage data to coveralls.io -# python: '3.6' -# env: TOXENV=reporting allow_failures: - env: TOXENV=quality @@ -72,70 +53,6 @@ after_failure: - more .tox/log/* | cat - more .tox/*/log/* | cat -# - stage: run tests and linters -# language: node_js -# node_js: 6 -# cache: yarn -# env: COMPONENT=server CMD=lint -# before_install: cd server -# script: bash ../scripts/travis-yarn.sh -# -#quality: &quality -# - stage: "Quality" -# name: "Quallity assertion" -# script: .travis/run-tox.sh -# -#tests: &tests -# - stage: "Tests" -# name: "Unit Tests" -# script: .travis/run-tox.sh - - -# - name: "Helper Tests" -# script: time ./run yarn test:named Helper -# - name: "Integration Tests" -# script: time ./run yarn test:named Integration -# - name: "Acceptance Tests" -# script: time ./run yarn test:named Acceptance -# - name: "a11y" -# script: PERCY_ENABLE=0 time ./run yarn test:named Acceptance --query enableA11yAudit=true - -#quality: &quality -# - stage: "Quality" -# name: "Lint JS/TS" -# script: time ./run yarn lint:js -# - name: "Lint Templates" -# script: time ./run yarn lint:hbs -# - name: "Lint Styles" -# script: time ./run yarn lint:sass -# - name: "Check Types" -# script: time ./run yarn tsc -# - name: "Translations" -# script: time ./run yarn lint:i18n - -#jobs: -# fail_fast: true -# -# include: -# - <<: *quality -# env: -# - TOXENV=quality - -# - <<: *tests -# env: -# - TOXENV=py35 -## -# - <<: *tests -# python: 3.6 -# env: -# - TOXENV=py36 -# -# - <<: *tests -# python: 3.7 -# env: -# - TOXENV=py37 - -#script: ./.travis/run-tox.sh # #deploy: # provider: script diff --git a/.travis/deploy.sh b/.travis/deploy.sh deleted file mode 100644 index e6af562..0000000 --- a/.travis/deploy.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -coveralls \ No newline at end of file diff --git a/.travis/install-tox.sh b/.travis/install-tox.sh deleted file mode 100644 index 82ef97f..0000000 --- a/.travis/install-tox.sh +++ /dev/null @@ -1,6 +0,0 @@ -!#/bin/bash - -set -e -set -u - -pip install tox diff --git a/.travis/run-tox.sh b/.travis/run-tox.sh deleted file mode 100644 index 17ef343..0000000 --- a/.travis/run-tox.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -e -set -u - -export TOX_SKIP_MISSING_INTERPRETERS="False"; - -tox - -coveralls \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 49c87ee..94fa08d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,37 +2,30 @@ description-file = README.rst license_file = LICENSE.txt - [flake8] max-line-length = 140 exclude = */migrations/* [tool:pytest] testpaths = tests -norecursedirs = - migrations - -python_files = - test_*.py - *_test.py - tests.py -addopts = - -ra - --strict - --doctest-modules - --doctest-glob=\*.rst - -python_versions = - py35 - py36 - py37 +norecursedirs = + migrations +python_files = + test_*.py + *_test.py + tests.py +addopts = + -ra + --strict + --doctest-modules + --doctest-glob=\*.rst +python_versions = + py35 + py36 + py37 +dependencies = +environment_variables = + - -dependencies = -# 1.4: Django==1.4.16 !python_versions[py3*] -# 1.5: Django==1.5.11 -# 1.6: Django==1.6.8 -# 1.7: Django==1.7.1 !python_versions[py26] -# Deps commented above are provided as examples. That's what you would use in a Django project. +[easy_install] -environment_variables = - - diff --git a/setup.py b/setup.py index 2adace7..bcf3d15 100644 --- a/setup.py +++ b/setup.py @@ -1,52 +1,78 @@ import os -from glob import glob -from os.path import basename -from os.path import splitext from setuptools import setup, find_packages my_dir = os.path.dirname(os.path.realpath(__file__)) + def readme(): with open(os.path.join(my_dir, 'README.rst')) as f: return f.read() + # return str(resource_string(__name__, 'README.rst')) setup( name='music_album_creation', - version='1.0.8a', - description='A CLI application intending to automate offline music library building', + version='1.1.0', + description='A CLI application intending to automate offline music library building.', long_description=readme(), - keywords=['music album', 'automation', 'youtube', 'audio metadata', 'download'], + keywords='music automation download youtube metadata', + + project_urls={ + "Source Code": "https://github.com/boromir674/music-album-creator", + }, + zip_safe=False, + + # what packages/distributions (python) need to be installed when this one is. (Roughly what is imported in source code) + install_requires=['tqdm', 'click', 'sklearn', 'mutagen', 'PyInquirer', 'youtube_dl'], + + # A string or list of strings specifying what other distributions need to be present in order for the setup script to run. + # (Note: projects listed in setup_requires will NOT be automatically installed on the system where the setup script is being run. + # They are simply downloaded to the ./.eggs directory if they’re not locally available already. If you want them to be installed, + # as well as being available when the setup script is run, you should add them to install_requires and setup_requires.) + # setup_requires=[], + + # Folder where unittest.TestCase-like written modules reside. Specifying this argument enables use of the test command + # to run the specified test suite, e.g. via setup.py test. + test_suite='tests', + + # Declare packages that the project’s tests need besides those needed to install it. A string or list of strings specifying + # what other distributions need to be present for the package’s tests to run. Note that these required projects will not be installed on the system where the + # tests are run, but only downloaded to the project’s setup directory if they’re not already installed locally. + # Use to ensure that a package is available when the test command is run. + tests_require=['pytest'], + classifiers=[ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Multimedia :: Sound/Audio :: Conversion', - 'Topic :: Multimedia :: Sound/Audio :: Editors', 'Intended Audience :: End Users/Desktop', - 'Intended Audience :: Science/Research' ], url='https://github.com/boromir674/music-album-creator', + download_url='point to tar.gz', # help easy_install do its tricks author='Konstantinos Lampridis', author_email='k.lampridis@hotmail.com', license='GNU GPLv3', packages=find_packages(where='src'), - package_dir={'': 'src'}, - py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], - install_requires=['tqdm', 'click', 'sklearn', 'mutagen', 'PyInquirer', 'youtube_dl'], - include_package_data=True, + package_dir={'': 'src'}, # this is required by distutils + # py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + include_package_data=True, # Include all data files in packages that distutils are aware of through the MANIFEST.in file + # package_data={ + # # If any package contains *.txt or *.rst files, include them: + # '': ['*.txt', '*.rst'], + # 'music_album_creation.format_classification': ['data/*.txt', 'data/model.pickle'], + # }, entry_points={ 'console_scripts': [ 'create-album = music_album_creation.create_album:main', ] }, - # TODO check if/where to put pytest - # setup_requires=['numpy>=1.11.0'], - tests_require=['tox', 'pytest'], - # test_suite='', - zip_safe=False + # A dictionary mapping names of “extras” (optional features of your project: eg imports that a console_script uses) to strings or lists of strings + # specifying what other distributions must be installed to support those features. + # extras_require={}, ) diff --git a/src/music_album_creation/dialogs.py b/src/music_album_creation/dialogs.py index 08e5a1c..6cd69e0 100644 --- a/src/music_album_creation/dialogs.py +++ b/src/music_album_creation/dialogs.py @@ -76,15 +76,15 @@ def multiline_input(prompt_=None): ##### STORE ALBUM DIALOG def store_album_dialog(tracks, music_lib='', artist='', album='', year=''): - def _copy_tracks(track_files, destination_directory): + def _copy_tracks(track_files, destination_dir): for track in track_files: - destination_file_path = os.path.join(destination_directory, os.path.basename(track)) + destination_file_path = os.path.join(destination_dir, os.path.basename(track)) if os.path.isfile(destination_file_path): print(" File '{}' already exists. in '{}'. Skipping".format(os.path.basename(track), - destination_directory)) + destination_dir)) else: shutil.copyfile(track, destination_file_path) - print("Album tracks reside in '{}'".format(destination_directory)) + print("Album tracks reside in '{}'".format(destination_dir)) if year: album = '{} ({})'.format(album, year) @@ -141,7 +141,7 @@ def validate(self, document): message='Please enter a number', cursor_position=len(document.text)) # Move cursor to end - def set_metadata_panel(artist=artist, album=album, year=year): + def set_metadata_panel(default_artist=artist, default_album=album, default_year=year): questions = [ { 'type': 'confirm', @@ -168,7 +168,7 @@ def set_metadata_panel(artist=artist, album=album, year=year): { 'type': 'input', 'name': 'artist', - 'default': artist, + 'default': default_artist, 'message': "'artist' tag", }, { @@ -180,14 +180,14 @@ def set_metadata_panel(artist=artist, album=album, year=year): { 'type': 'input', 'name': 'album', - 'default': album, + 'default': default_album, 'message': "'album' tag", }, { 'type': 'input', 'name': 'year', 'message': "'year' tag", - 'default': year, # trick to allow empty value + 'default': default_year, # trick to allow empty value 'validate': NumberValidator, # 'filter': lambda val: int(val) }, diff --git a/tox.ini b/tox.ini index 469b31d..63a8116 100644 --- a/tox.ini +++ b/tox.ini @@ -5,10 +5,7 @@ envlist = py35, py36, py37, - report, -; coveralls -; reporting -; report + reporting, ; coveralls @@ -17,11 +14,10 @@ skip_missing_interpreters = {env:TOX_SKIP_MISSING_INTERPRETERS:True} [testenv] basepython = {docs,spell}: {env:TOXPYTHON:python3.6} - {bootstrap,clean,check,report,coveralls,quality}: {env:TOXPYTHON:python3} + {bootstrap,clean,check,reporting,coveralls,quality}: {env:TOXPYTHON:python3} setenv = PYTHONPATH={toxinidir}/tests PYTHONUNBUFFERED=yes - # klein project below 2 PIP_DISABLE_PIP_VERSION_CHECK=1 VIRTUALENV_NO_DOWNLOAD=1 passenv = @@ -32,11 +28,11 @@ passenv = codecov: TRAVIS TRAVIS_* deps = pytest - # pytest-travis-fold pytest-cov commands = - {posargs:pytest --cov --cov-report=term-missing -vv --ignore=src} + # --cov-append so that multiple test runs do not erase the .coverage file at each of their starts + {posargs:pytest --cov --cov-report=term-missing -vv --ignore=src --cov-append} # This env must succeed in order to be meaningful to proceed running the rest of the environments @@ -101,7 +97,7 @@ ignore = W504, # line too long E501, - # class GavError(Exception): pass + # multiple statements on one line (colon) E701, # too many leading # for block comment E266, @@ -112,18 +108,8 @@ ignore = [testenv:reporting] -basepython = {env:TOXPYTHON:python3.6} deps = coverage - coveralls -skip_install = true -commands = - coverage report - coverage html - coveralls - -[testenv:report] -deps = coverage skip_install = true commands = coverage report @@ -134,9 +120,10 @@ deps = coveralls skip_install = true commands = +# requires COVERALLS_REPO_TOKEN coveralls [] +;; coveralls (same as above) ;; coverage run --source=music_album_creation setup.py test -;; coveralls [testenv:py35] @@ -147,19 +134,3 @@ basepython = {env:TOXPYTHON:python3.6} [testenv:py37] basepython = {env:TOXPYTHON:python3.7} - - - -;[testenv:quality] -;basepython = {env:TOXPYTHON:python3.6} -;deps = -; docutils -; check-manifest -; coverage -; flake8 -; readme-renderer -; pygments -;skip_install = true -;commands = -; python setup.py check --strict --metadata --restructuredtext -; flake8 src tests setup.py