From 8275d7e7b8bcb60426f04064447530fde651590f Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Mon, 13 May 2019 23:34:17 +0300 Subject: [PATCH] added R-package docs generation routines --- .appveyor.yml | 57 -------------- .readthedocs.yml | 11 +++ .travis.yml | 65 ---------------- .vsts-ci.yml | 164 ---------------------------------------- R-package/.Rbuildignore | 2 + R-package/_pkgdown.yml | 124 ++++++++++++++++++++++++++++++ build_r.R | 2 +- build_r_site.R | 23 ++++++ docs/R-API.rst | 4 + docs/conf.py | 50 +++++++++++- docs/index.rst | 1 + 11 files changed, 215 insertions(+), 288 deletions(-) delete mode 100644 .appveyor.yml create mode 100644 .readthedocs.yml delete mode 100644 .travis.yml delete mode 100644 .vsts-ci.yml create mode 100644 R-package/_pkgdown.yml create mode 100644 build_r_site.R create mode 100644 docs/R-API.rst diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index dac39ec25567..000000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,57 +0,0 @@ -version: 2.2.4.{build} - -image: Visual Studio 2015 -platform: x64 -configuration: # a trick to construct a build matrix with multiple Python versions - - 3.6 - -environment: - matrix: - - COMPILER: MSVC - - COMPILER: MINGW - -clone_depth: 50 - -install: - - git submodule update --init --recursive # get `compute` folder - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - ps: >- - switch ($env:PYTHON_VERSION) { - "2.7" {$env:MINICONDA = """C:\Miniconda-x64"""} - "3.5" {$env:MINICONDA = """C:\Miniconda35-x64"""} - "3.6" {$env:MINICONDA = """C:\Miniconda36-x64"""} - "3.7" {$env:MINICONDA = """C:\Miniconda37-x64"""} - default {$env:MINICONDA = """C:\Miniconda37-x64"""} - } - - set PATH=%MINICONDA%;%MINICONDA%\Scripts;%PATH% - - ps: $env:LGB_VER = (Get-Content VERSION.txt).trim() - - conda config --set always_yes yes --set changeps1 no - - conda update -q -y conda - - conda create -q -y -n test-env python=%PYTHON_VERSION% matplotlib numpy pandas psutil pytest python-graphviz scikit-learn scipy - - activate test-env - - set PATH=%CONDA_PREFIX%\Library\bin\graphviz;%PATH% # temp graphviz hotfix - -build_script: - - cd %APPVEYOR_BUILD_FOLDER%\python-package - - IF "%COMPILER%"=="MINGW" ( - python setup.py install --mingw) - ELSE ( - python setup.py install) - -test_script: - - pytest %APPVEYOR_BUILD_FOLDER%\tests\python_package_test - - cd %APPVEYOR_BUILD_FOLDER%\examples\python-guide - - ps: >- - @("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py" # prevent interactive window mode - (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py" - - ps: >- - foreach ($file in @(Get-ChildItem *.py)) { - @("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file - python $file - if (!$?) { $host.SetShouldExit(-1) } - } # run all examples - - cd %APPVEYOR_BUILD_FOLDER%\examples\python-guide\notebooks - - conda install -q -y -n test-env ipywidgets notebook - - jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb # run all notebooks diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 000000000000..c2e19847a4fc --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,11 @@ +version: 2 +formats: + - pdf +python: + version: 3 + install: + - requirements: docs/requirements.txt +sphinx: + builder: html + configuration: docs/conf.py + fail_on_warning: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c2c2a1ef7b12..000000000000 --- a/.travis.yml +++ /dev/null @@ -1,65 +0,0 @@ -language: cpp - -git: - submodules: true - -os: - - linux - - osx -dist: trusty -osx_image: xcode10.2 - -env: - global: # default values - - PYTHON_VERSION=3.7 - matrix: - - TASK=regular PYTHON_VERSION=3.6 - - TASK=sdist PYTHON_VERSION=2.7 - - TASK=bdist - - TASK=pylint - - TASK=check-docs - - TASK=mpi METHOD=source - - TASK=mpi METHOD=pip - - TASK=gpu METHOD=source PYTHON_VERSION=3.5 - - TASK=gpu METHOD=pip PYTHON_VERSION=3.6 - -matrix: - exclude: - - os: osx - env: TASK=gpu METHOD=source PYTHON_VERSION=3.5 - - os: osx - env: TASK=gpu METHOD=pip PYTHON_VERSION=3.6 - - os: osx - env: TASK=pylint - - os: osx - env: TASK=check-docs - -before_install: - - test -n $CC && unset CC - - test -n $CXX && unset CXX - - export HOME_DIRECTORY="$HOME" - - export BUILD_DIRECTORY="$TRAVIS_BUILD_DIR" - - if [[ $TRAVIS_OS_NAME == "osx" ]]; then - export OS_NAME="macos"; - export COMPILER="gcc"; - else - export OS_NAME="linux"; - export COMPILER="clang"; - fi - - export CONDA="$HOME/miniconda" - - export PATH="$CONDA/bin:$PATH" - - export CONDA_ENV="test-env" - - export LGB_VER=$(head -n 1 VERSION.txt) - - export AMDAPPSDK_PATH=$HOME/AMDAPPSDK - - export LD_LIBRARY_PATH="$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH" - - export LD_LIBRARY_PATH="/usr/local/clang/lib:$LD_LIBRARY_PATH" # fix error "libomp.so: cannot open shared object file: No such file or directory" on Linux with Clang - - export OPENCL_VENDOR_PATH=$AMDAPPSDK_PATH/etc/OpenCL/vendors - -install: - - bash .ci/setup.sh - -script: - - bash .ci/test.sh - -notifications: - email: false diff --git a/.vsts-ci.yml b/.vsts-ci.yml deleted file mode 100644 index 1f52e0ec8bcb..000000000000 --- a/.vsts-ci.yml +++ /dev/null @@ -1,164 +0,0 @@ -variables: - PYTHON_VERSION: 3.7 - CONDA_ENV: test-env -resources: - containers: - - container: ubuntu1404 - image: lightgbm/vsts-agent:ubuntu-14.04 -jobs: -########################################### -- job: Linux -########################################### - variables: - COMPILER: gcc - pool: - vmImage: 'ubuntu-16.04' - container: ubuntu1404 - strategy: - maxParallel: 7 - matrix: - regular: - TASK: regular - sdist: - TASK: sdist - PYTHON_VERSION: 3.5 - bdist: - TASK: bdist - PYTHON_VERSION: 3.6 - swig: - TASK: swig - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - PYTHON_VERSION: 2.7 - gpu_source: - TASK: gpu - METHOD: source - PYTHON_VERSION: 3.6 - steps: - - script: | - echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.setvariable variable=OS_NAME]linux" - echo "##vso[task.setvariable variable=AZURE]true" - echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - echo "##vso[task.prependpath]$CONDA/bin" - AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK - echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" - LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH - echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" - echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" - displayName: 'Set variables' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container -########################################### -- job: MacOS -########################################### - variables: - COMPILER: clang - pool: - vmImage: 'macOS-10.13' - strategy: - maxParallel: 3 - matrix: - regular: - TASK: regular - PYTHON_VERSION: 3.6 - sdist: - TASK: sdist - PYTHON_VERSION: 3.5 - bdist: - TASK: bdist - steps: - - script: | - echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.setvariable variable=OS_NAME]macos" - echo "##vso[task.setvariable variable=AZURE]true" - echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - CONDA=$AGENT_HOMEDIRECTORY/miniconda - echo "##vso[task.setvariable variable=CONDA]$CONDA" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container -########################################### -- job: Windows -########################################### - pool: - vmImage: 'vs2017-win2016' - strategy: - maxParallel: 3 - matrix: - regular: - TASK: regular - PYTHON_VERSION: 3.5 - sdist: - TASK: sdist - PYTHON_VERSION: 2.7 - bdist: - TASK: bdist - PYTHON_VERSION: 3.6 - steps: - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Enable conda - - script: | - conda update -q -y conda - conda create -q -y -n %CONDA_ENV% python=%PYTHON_VERSION% matplotlib numpy pandas psutil pytest python-graphviz scikit-learn scipy - cmd /c "activate %CONDA_ENV% & powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1" - displayName: Test - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container - -########################################### -- job: Package -########################################### - dependsOn: - - Linux - - MacOS - - Windows - condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - pool: - vmImage: 'vs2017-win2016' - steps: - # Download all agent packages from all previous phases - - task: DownloadBuildArtifacts@0 - displayName: Download package assets - inputs: - artifactName: PackageAssets - downloadPath: $(Build.SourcesDirectory)/binaries - - script: | - python %BUILD_SOURCESDIRECTORY%/.nuget/create_nuget.py %BUILD_SOURCESDIRECTORY%/binaries/PackageAssets - displayName: 'Create NuGet configuration files' - - task: NuGetCommand@2 - inputs: - command: pack - packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec' - packDestination: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: NuGet - artifactType: container diff --git a/R-package/.Rbuildignore b/R-package/.Rbuildignore index 193677ab5a01..1fb7a359298e 100644 --- a/R-package/.Rbuildignore +++ b/R-package/.Rbuildignore @@ -1,5 +1,7 @@ ^build_package.R$ \.gitkeep$ +^docs$ +^_pkgdown\.yml$ # Objects created by compilation \.o$ diff --git a/R-package/_pkgdown.yml b/R-package/_pkgdown.yml new file mode 100644 index 000000000000..9cf6f6d8ad66 --- /dev/null +++ b/R-package/_pkgdown.yml @@ -0,0 +1,124 @@ +template: + params: + bootswatch: united + +authors: + Guolin Ke: + href: https://github.com/guolinke + html: Guolin Ke + Damien Soukhavong: + href: https://github.com/Laurae2 + html: Damien Soukhavong + Yachen Yan: + href: https://github.com/yanyachen + html: Yachen Yan + James Lamb: + href: https://github.com/jameslamb + html: James Lamb + + +site: + root: '' + title: LightGBM, Light Gradient Boosting Machine + +reference: + - title: Dataset + desc: Datasets included with the R package + contents: + - '`agaricus.test`' + - '`agaricus.train`' + - '`bank`' + - title: Data Input / Output + desc: Data I/O required for LightGBM + contents: + - '`dim.lgb.Dataset`' + - '`dimnames.lgb.Dataset`' + - '`getinfo`' + - '`setinfo`' + - '`slice`' + - '`lgb.Dataset.construct`' + - '`lgb.Dataset.create.valid`' + - '`lgb.Dataset`' + - '`lgb.Dataset.save`' + - '`lgb.Dataset.set.categorical`' + - '`lgb.Dataset.set.reference`' + - title: Machine Learning + desc: Train models with LightGBM + contents: + - '`lgb.prepare`' + - '`lgb.prepare2`' + - '`lgb.prepare_rules`' + - '`lgb.prepare_rules2`' + - '`lgb.cv`' + - '`lgb.train`' + - title: Saving / Loading Models + desc: Save and Load LightGBM models + contents: + - '`lgb.dump`' + - '`lgb.load`' + - '`lgb.model.dt.tree`' + - '`lgb.save`' + - '`predict.lgb.Booster`' + - '`readRDS.lgb.Booster`' + - '`saveRDS.lgb.Booster`' + - title: Predictive Analysis + desc: Analyze your predictions + contents: + - '`lgb.get.eval.result`' + - '`lgb.importance`' + - '`lgb.interprete`' + - '`lgb.plot.importance`' + - '`lgb.plot.interpretation`' + - title: Miscellaneous + desc: Ungroupable functions to troubleshoot LightGBM + contents: + - '`lgb.unloader`' + +navbar: + title: LightGBM + type: default + left: + - icon: fa-home fa-lg + href: index.html + - text: Reference + href: reference/index.html + - text: Vignettes + menu: + - text: Basic Walkthrough + href: articles/basic_walkthrough.html + - text: Boosting from existing prediction + href: articles/boost_from_prediction.html + - text: Categorical Feature Preparation + href: articles/categorical_features_prepare.html + - text: Categorical Feature Preparation with Rule + href: articles/categorical_features_rules.html + - text: Cross Validation + href: articles/cross_validation.html + - text: Early Stop in training + href: articles/early_stopping.html + - text: Efficiency for Many Model Trainings + href: articles/efficient_many_training.html + - text: Leaf (in)Stability example + href: articles/leaf_stability.html + - text: Multiclass training/prediction + href: articles/multiclass.html + - text: Weight-Parameter adjustment relationship + href: articles/weight_param.html + right: + - icon: fa-github fa-lg + href: https://github.com/Microsoft/LightGBM + +articles: +- title: Vignettes + desc: ~ + contents: + - '`basic_walkthrough`' + - '`boost_from_prediction`' + - '`categorical_features_prepare`' + - '`categorical_features_rules`' + - '`cross_validation`' + - '`early_stopping`' + - '`efficient_many_training`' + - '`leaf_stability`' + - '`multiclass`' + - '`weight_param`' diff --git a/build_r.R b/build_r.R index d6d2d0e17d01..f69f34c042ce 100644 --- a/build_r.R +++ b/build_r.R @@ -56,7 +56,7 @@ result <- file.copy(from = "CMakeLists.txt", overwrite = TRUE) .handle_result(result) -# Build the package +# Build the package (do not touch this line!) # NOTE: --keep-empty-dirs is necessary to keep the deep paths expected # by CMake while also meeting the CRAN req to create object files # on demand diff --git a/build_r_site.R b/build_r_site.R new file mode 100644 index 000000000000..3538daee8f54 --- /dev/null +++ b/build_r_site.R @@ -0,0 +1,23 @@ +setwd("/home/docs/checkouts/readthedocs.org/user_builds/lightgbm/checkouts/docs/lightgbm_r") + +if (!dir.exists("./docs")) { + dir.create("./docs") +} + +print("========================building pkgdown site====================================") +# options(pkgdown.internet=FALSE) +library(pkgdown) + +clean_site() +init_site() +build_home(quiet = FALSE, preview = FALSE) +build_reference(document = TRUE, preview = FALSE) +# # to-do +# build_articles(preview = FALSE) +# build_tutorials(preview = FALSE) +# build_news(preview = FALSE) + +# # don't work +# pkgdown::build_site(pkg = ".", examples = FALSE, document = TRUE, +# run_dont_run = TRUE, seed = 1014, lazy = FALSE, +# override = list(), preview = NA, new_process = FALSE) diff --git a/docs/R-API.rst b/docs/R-API.rst new file mode 100644 index 000000000000..38486ea62251 --- /dev/null +++ b/docs/R-API.rst @@ -0,0 +1,4 @@ +R API +===== + +Refer to https://lightgbm.readthedocs.io/en/docs/R/reference. diff --git a/docs/conf.py b/docs/conf.py index 99b5d1dc4894..b15ebee14ed9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,6 +22,7 @@ import sys import sphinx +from distutils.dir_util import copy_tree from docutils.parsers.rst import Directive from sphinx.errors import VersionRequirementError from subprocess import PIPE, Popen @@ -56,6 +57,8 @@ def run(self): os.environ['LIGHTGBM_BUILD_DOC'] = '1' C_API = os.environ.get('C_API', '').lower().strip() != 'no' +RTD = bool(os.environ.get('READTHEDOCS', '')) +R_API = RTD # If your documentation needs a minimal Sphinx version, state it here. needs_sphinx = '1.3' # Due to sphinx.ext.napoleon @@ -208,6 +211,44 @@ def generate_doxygen_xml(app): raise Exception("An error has occurred while executing Doxygen\n" + str(e)) +def generate_r_docs(app): + """Generate documentation for R-package. + + Parameters + ---------- + app : object + The application object representing the Sphinx process. + """ + commands = """ + export PATH="/home/docs/.conda/bin:$PATH" + echo 'options(repos = "https://cran.rstudio.com")' > $HOME/.Rprofile + conda create -q -y -n r_env r-base r-devtools r-data.table r-jsonlite r-magrittr r-matrix r-testthat cmake + conda install -q -y -n r_env -c conda-forge r-pkgdown + source activate r_env + export TAR=/bin/tar + cd /home/docs/checkouts/readthedocs.org/user_builds/lightgbm/checkouts/docs + sed -i'.bak' '/# Build the package (do not touch this line!)/q' build_r.R + Rscript build_r.R + Rscript build_r_site.R + """ + try: + # Warning! The following code can cause buffer overflows on RTD. + # Consider suppressing output completely if RTD project silently fails. + # Refer to https://github.com/svenevs/exhale + # /blob/fe7644829057af622e467bb529db6c03a830da99/exhale/deploy.py#L99-L111 + process = Popen(['/bin/bash'], + stdin=PIPE, stdout=PIPE, stderr=PIPE, + universal_newlines=True) + stdout, stderr = process.communicate(commands) + output = '\n'.join([i for i in (stdout, stderr) if i is not None]) + if process.returncode != 0: + raise RuntimeError(output) + else: + print(output) + except BaseException as e: + raise Exception("An error has occurred while generating documentation for R-package\n" + str(e)) + + def setup(app): """Add new elements at Sphinx initialization time. @@ -216,8 +257,15 @@ def setup(app): app : object The application object representing the Sphinx process. """ - if C_API: + first_run = not os.path.exists(os.path.join(CURR_PATH, '_FIRST_RUN.flag')) + if first_run and RTD: + open(os.path.join(CURR_PATH, '_FIRST_RUN.flag'), 'w').close() + if C_API and (not RTD or first_run): app.connect("builder-inited", generate_doxygen_xml) else: app.add_directive('doxygenfile', IgnoredDirective) + if R_API: + if not RTD or first_run: + app.connect("builder-inited", generate_r_docs) + app.connect("build-finished", lambda app, exception: copy_tree(app.confdir + '/../lightgbm_r/docs', app.outdir + '/R', verbose=0)) app.add_javascript("js/script.js") diff --git a/docs/index.rst b/docs/index.rst index 5ae58fc2b9f1..3a2574039bbd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,6 +29,7 @@ For more details, please refer to `Features <./Features.rst>`__. Parameters Tuning C API Python API + R API Parallel Learning Guide GPU Tutorial Advanced Topics