Aggregate documentation

.github/workflows/docs.yml

@@ -0,0 +1,37 @@
+name: Docs
+on:
+  push:
+    branches:
+      - main
+    tags: '*'
+  pull_request:
+jobs:
+  docs:
+    name: Documentation
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: '1'
+      - uses: julia-actions/cache@v2
+      # Round 1: Create interlink dumps
+      - run: Core/docs/make.jl
+      - run: Store/docs/make.jl
+      - run: Common/docs/make.jl
+      - run: REPL/docs/make.jl
+      - run: Main/docs/make.jl
+      # Round 2: Now with resolvable interlinks
+      - run: Core/docs/make.jl --only-if-inv-changed
+      - run: Store/docs/make.jl --only-if-inv-changed
+      - run: Common/docs/make.jl --only-if-inv-changed
+      - run: REPL/docs/make.jl --only-if-inv-changed
+      # Combine
+      - name: MultiDocs
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          julia --project=MultiDocs -e 'using Pkg; Pkg.instantiate()'
+          julia --project=MultiDocs MultiDocs/make-multi.jl

Base/.gitignore

@@ -0,0 +1,3 @@
+/Manifest.toml
+/docs/src/**.md
+/docs/build

Base/Project.toml

@@ -0,0 +1,9 @@
+name = "DataToolkitBase"
+uuid = "e209d0c3-e863-446f-9b45-de6ca9730756"
+authors = ["TEC <[email protected]>"]
+version = "0.10.0"
+
+[deps]
+DataToolkitCommon = "9e6fccbf-6142-406a-aa4c-75c1ae647f53"
+DataToolkitCore = "caac3e55-418c-402e-a061-64d454aa8f4f"
+DataToolkitStore = "082ec3c2-3fb3-458f-ad22-5e5e31d4377a"

Base/src/DataToolkitBase.jl

@@ -0,0 +1,72 @@
+module DataToolkitBase
+
+using DataToolkitCore
+using DataToolkitCommon
+using DataToolkitStore
+
+export @d_str, @load, @require, @addpkg
+
+const var"@require" = DataToolkitCore.var"@require"
+const var"@addpkg" = DataToolkitCore.var"@addpkg"
+
+"""
+    @d_str -> loaded data
+
+Shorthand for loading a dataset in the default format,
+`d"iris"` is equivalent to `read(dataset("iris"))`.
+"""
+macro d_str(ident::String)
+    quote
+        ref = parse(Identifier, $ident)
+        if !isnothing(ref.type)
+            resolve(ref)
+        else
+            read(resolve(ref, resolvetype=false))
+        end
+    end
+end
+
+function loadproject!(mod::Module, projpath::String; force::Bool=false)
+    function tryloadcollection!(path::String, m::Module; soft::Bool)
+        try
+            loadcollection!(path, m; soft)
+        catch err
+            @error "Failed to load $path" err
+        end
+    end
+    if !isdir(projpath)
+        projpath = dirname(projpath)
+    end
+    # Skip packages when `init(Main)` called.
+    if mod === Main && isfile(joinpath(projpath, "Project.toml"))
+        data = Base.parsed_toml(joinpath(projpath, "Project.toml"))
+        ispkg = haskey(data, "name") && haskey(data, "uuid") &&
+            haskey(data, "version") && isfile(joinpath(
+                projpath, "src", data["name"] * ".jl"))
+        ispkg && return
+    end
+    # Load Data.d/*.toml
+    data_dir = joinpath(projpath, "Data.d")
+    if isdir(data_dir)
+        dfiles = filter(f -> endswith(f, ".toml"),
+                        readdir(data_dir, join=true))
+        for dfile in filter(f -> basename(f) != "Data.toml", dfiles)
+            tryloadcollection!(dfile, mod, soft=!force)
+        end
+        # Load Data.toml last so that is is first in the stack.
+        joinpath(data_dir, "Data.toml") in dfiles &&
+            tryloadcollection!("Data.d/Data.toml", mod, soft=!force)
+    end
+    # Load Data.toml
+    data_file = joinpath(projpath, "Data.toml")
+    if isfile(data_file)
+        isdir(data_dir) && @warn "($mod) consider placing Data.toml file inside Data.d directory"
+        tryloadcollection!(data_file, mod, soft=!force)
+    end
+end
+
+macro load()
+    :(loadproject!(@__MODULE__, pkgdir(@__MODULE__)))
+end
+
+end

Common/.github/workflows/CompatHelper.yml

@@ -0,0 +1,16 @@
+name: CompatHelper
+on:
+  schedule:
+    - cron: 0 0 * * *
+  workflow_dispatch:
+jobs:
+  CompatHelper:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pkg.add("CompatHelper")
+        run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
+      - name: CompatHelper.main()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}
+        run: julia -e 'using CompatHelper; CompatHelper.main()'

Common/.github/workflows/TagBot.yml

@@ -0,0 +1,15 @@
+name: TagBot
+on:
+  issue_comment:
+    types:
+      - created
+  workflow_dispatch:
+jobs:
+  TagBot:
+    if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JuliaRegistries/TagBot@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ssh: ${{ secrets.DOCUMENTER_KEY }}

Common/.github/workflows/ci.yml

@@ -0,0 +1,84 @@
+name: CI
+on:
+  push:
+    branches:
+      - main
+    tags: '*'
+  pull_request:
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.version == 'nightly' }}
+    strategy:
+      matrix:
+        version:
+          - '1.8'
+          - '1.9'
+          - '1' # 1.10
+          - 'nightly'
+        os:
+          - ubuntu-latest
+          # - macOS-latest
+          # - windows-latest
+        arch:
+          - x64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: julia-actions/cache@v1
+      - uses: julia-actions/julia-buildpkg@latest
+      - uses: julia-actions/julia-runtest@latest
+  test-dev:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} (dev)
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.version == 'nightly' }}
+    strategy:
+      matrix:
+        version:
+          - '1.9'
+          - '1'
+          - 'nightly'
+        os:
+          - ubuntu-latest
+          # - macOS-latest
+          # - windows-latest
+        arch:
+          - x64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: julia-actions/cache@v1
+      - run: julia --project=. -e '
+          using Pkg;
+          Pkg.add(name="DataToolkitCore", rev="main");
+          Pkg.compat("DataToolkitCore", ">=0")'
+      - uses: julia-actions/julia-buildpkg@latest
+      - uses: julia-actions/julia-runtest@latest
+  docs:
+    name: Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: '1'
+      - if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        run: julia --project=docs -e '
+          using Pkg;
+          Pkg.develop(PackageSpec(; path=pwd()));
+          Pkg.add(name="DataToolkitCore", rev="main");
+          Pkg.compat("DataToolkitCore", ">=0")'
+      - run: julia --project=docs -e '
+          using Pkg;
+          Pkg.develop(PackageSpec(; path=pwd()));
+          Pkg.instantiate();'
+      - run: julia --project=docs docs/make.jl
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Common/.gitignore

@@ -0,0 +1,3 @@
+/Manifest.toml
+/docs/src/**.md
+/docs/build/

Common/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 TEC <[email protected]> and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Common/Project.toml

@@ -0,0 +1,129 @@
+name = "DataToolkitCommon"
+uuid = "9e6fccbf-6142-406a-aa4c-75c1ae647f53"
+version = "0.10.0"
+authors = ["TEC <[email protected]> and contributors"]
+
+[deps]
+BaseDirs = "18cc8868-cbac-4acf-b575-c8ff214dc66f"
+Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
+DataToolkitCore = "caac3e55-418c-402e-a061-64d454aa8f4f"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
+UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[weakdeps]
+AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
+ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3"
+Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd"
+CodecXz = "ba30903b-d9e8-5048-a5ec-d1f5b0d4b47b"
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
+CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
+ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
+DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965"
+DataToolkitStore = "082ec3c2-3fb3-458f-ad22-5e5e31d4377a"
+DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
+GIFImages = "7064036c-d33e-4a25-b247-cf6150d8ad81"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+JpegTurbo = "b835a17e-a41a-41e7-81f0-2f016b05efe0"
+Netpbm = "f09324ee-3d7c-5217-9330-fc30815ba969"
+PNGFiles = "f57f5aa1-a3ce-4bc8-8ab9-96f992907883"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+QOI = "4b34888f-f399-49d4-9bb3-47ed5cae4e65"
+SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+TiffImages = "731e570b-9d59-4bfa-96dc-6df516fadf69"
+WebP = "e3aaa7dc-3e4b-44e0-be63-ffb868ccd7c1"
+XLSX = "fdbf4ff8-1666-58a4-91e7-1b58723a45e0"
+YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
+ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
+
+[extensions]
+AWSExt = "AWS"
+ArchGDALExt = "ArchGDAL"
+ArrowExt = "Arrow"
+CSVExt = "CSV"
+CodecBzip2Ext = "CodecBzip2"
+CodecXzExt = "CodecXz"
+CodecZlibExt = "CodecZlib"
+CodecZstdExt = "CodecZstd"
+DelimitedFilesExt = "DelimitedFiles"
+DownloadsExt = "Downloads"
+FilePathsBaseExt = "FilePathsBase"
+GIFImagesExt = "GIFImages"
+JLD2Ext = "JLD2"
+JSON3Ext = "JSON3"
+JpegTurboExt = ["ColorTypes", "JpegTurbo"]
+NetpbmExt = "Netpbm"
+PNGFilesExt = "PNGFiles"
+PkgExt = "Pkg"
+QOIExt = "QOI"
+SQLiteExt = "SQLite"
+SQLiteQueryExt = ["DBInterface", "SQLite"]
+DataToolkitStoreExt = "DataToolkitStore"
+TarExt = "Tar"
+TiffImagesExt = "TiffImages"
+WebPExt = "WebP"
+XLSXExt = "XLSX"
+YAMLExt = "YAML"
+ZipFileExt = "ZipFile"
+
+[compat]
+BaseDirs = "1"
+CRC32c = "1"
+Compat = "3.29, 4"
+DataToolkitCore = "0.10"
+Dates = "1"
+Markdown = "1"
+PrecompileTools = "1"
+Serialization = "1"
+TOML = "1"
+Tables = "1"
+UUIDs = "1"
+XLSX = "0.9,0.10"
+julia = "1.9"
+
+[extras]
+ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3"
+Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
+CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc"
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd"
+CodecXz = "ba30903b-d9e8-5048-a5ec-d1f5b0d4b47b"
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
+CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
+ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
+DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+DataToolkitCore = "e209d0c3-e863-446f-9b45-de6ca9730756"
+DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+JpegTurbo = "b835a17e-a41a-41e7-81f0-2f016b05efe0"
+KangarooTwelve = "2a5dabf5-6a39-42aa-818d-ce8a58d1b312"
+MD5 = "6ac74813-4b46-53a4-afec-0b5dc9d7885c"
+Netpbm = "f09324ee-3d7c-5217-9330-fc30815ba969"
+PNGFiles = "f57f5aa1-a3ce-4bc8-8ab9-96f992907883"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+QOI = "4b34888f-f399-49d4-9bb3-47ed5cae4e65"
+SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TiffImages = "731e570b-9d59-4bfa-96dc-6df516fadf69"
+XLSX = "fdbf4ff8-1666-58a4-91e7-1b58723a45e0"
+YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
+ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
+
+[targets]
+test = ["AWS", "ArchGDAL", "Arrow", "CRC32c", "CSV", "CodecBzip2", "CodecXz", "CodecZlib", "CodecZstd", "ColorTypes", "DBInterface", "DataFrames", "DataToolkitCore", "DelimitedFiles", "Downloads", "JLD2", "JSON3", "JpegTurbo", "KangarooTwelve", "MD5", "Netpbm", "PNGFiles", "Pkg", "QOI", "SHA", "SQLite", "Tar", "Test", "TiffImages", "XLSX", "YAML", "ZipFile"]