docs: revamp outline (secretflow#156)

.gitignore

@@ -24,4 +24,6 @@ logs/scdbserver.log
 *.mo
 *.pot
 
-*.pyc
+*.pyc
+
+.venv

cmd/docgen/main.go

@@ -132,7 +132,7 @@ func main() {
 		}).ParseFiles(fileName)
 	check(err)
 
-	f, err := os.Create("docs/reference/scql_operators.md")
+	f, err := os.Create("docs/reference/operators.md")
 	check(err)
 	err = tmpl.Execute(f, filler)
 	check(err)

cmd/docgen/scql_operators.md.tmpl

@@ -1,3 +1,9 @@
+# SCQL Operators Specification
+
+This is a specification (not a kernel library) of SCQL operators, including operator signatures and semantics.
+
+## Op List
+
 <!-- Autogenerated by docgen; don't manually edit -->
 {{- /*gotype: github.com/secretflow/scql/cmd/docgen.OpDocFiller*/ -}}
 

docs/conf.py

@@ -29,7 +29,6 @@
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
-
 # -- Project information -----------------------------------------------------
 
 project = "SCQL"
@@ -43,21 +42,20 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    "sphinx.ext.napoleon",
-    "sphinx.ext.autodoc",
-    "sphinx.ext.todo",
-    "sphinx.ext.viewcode",
-    "sphinx.ext.extlinks",
-    "sphinx.ext.autosectionlabel",
     "myst_parser",
     "nbsphinx",
-    "sphinxcontrib.actdiag",
-    "sphinxcontrib.blockdiag",
-    "sphinxcontrib.nwdiag",
-    "sphinxcontrib.packetdiag",
-    "sphinxcontrib.rackdiag",
-    "sphinxcontrib.seqdiag",
+    "secretflow_doctools",
     "sphinx_design",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosectionlabel",
+    "sphinx.ext.extlinks",
+    "sphinx.ext.graphviz",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.todo",
+    "sphinx.ext.viewcode",
+    "sphinxcontrib.mermaid",
 ]
 
 # Make sure the target is unique
@@ -77,6 +75,7 @@
 locale_dirs = ["./locales/"]  # path is example but recommended.
 gettext_compact = False  # optional.
 gettext_uuid = False  # optional.
+gettext_allow_fuzzy_translations = True
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -87,6 +86,10 @@
 # Enable TODO
 todo_include_todos = True
 
+# The output format for Graphviz when building HTML files. This must be either
+# 'png' or 'svg'; the default is 'png'.
+graphviz_output_format = "svg"
+
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -139,19 +142,9 @@
     "tasklist",
 ]
 
-suppress_warnings = ["myst.header"]
+suppress_warnings = ["autosectionlabel", "myst.header"]
 
-myst_gfm_only = True
-myst_heading_anchors = 1
-myst_title_to_header = True
-
-
-# app setup hook
-def setup(app):
-    app.add_config_value(
-        "recommonmark_config",
-        {
-            "auto_toc_tree_section": "Contents",
-        },
-        True,
-    )
+# Enable all MyST features
+myst_gfm_only = False
+# Enable anchors for heading level h1 through h6
+myst_heading_anchors = 6

docs/index.rst

@@ -9,10 +9,75 @@ SCQL Documentation
 Secure Collaborative Query Language (SCQL) is a system that translates SQL statements into a hybrid MPC-plaintext execution graph and executes them on a federation of database systems. The MPC framework is powered by `SPU <https://github.com/secretflow/spu>`_.
 
 
+Getting started
+---------------
+
+Follow the :doc:`tutorial </intro/tutorial>` and try out SCQL on your machine!
+
+
+SCQL Systems
+------------
+
+- **Overview**:
+  :doc:`System overview and architecture </topics/system/intro>` |
+  :doc:`Common usage </topics/system/usage>`
+
+- **Security**:
+  :doc:`Security overview </topics/security/overview>`
+
+- **Reference**:
+  :doc:`SCQL implementation status </reference/implementation-status>`
+
+
+The SCQL Language
+-----------------
+
+- **Reference**:
+  :doc:`SCQL language manual </reference/lang/manual>` |
+  :doc:`Compatibility with MySQL </reference/lang/mysql-compatibility>`
+
+
+Column Control List (CCL)
+-------------------------
+
+- **Overview**:
+  :doc:`Introduction to CCL </topics/ccl/intro>`
+
+- **Guides**:
+  :doc:`Common usage and advice </topics/ccl/usage>`
+
+- **Reference**:
+  :ref:`GRANT and REVOKE <scql_grant_revoke>` |
+  :ref:`How CCL works <how_ccl_works>`
+
+
+Clients
+-------
+
+- :doc:`Overview of SCQL clients </topics/clients/overview>`
+- :doc:`HTTP API reference </reference/http-api>`
+
+
+Deployment
+----------
+
+- **Guides**:
+  :doc:`How to deploy an SCQL cluster </topics/deployment/how-to-deploy-a-cluster>`
+
+- **Reference**:
+  :doc:`SCQL system config </reference/engine-config>`
+
+
+For contributors
+----------------
+
+- **Reference**:
+  :doc:`SCQL operators </reference/operators>`
+
+
 .. toctree::
-   :maxdepth: 2
-   :hidden:
-
-   getting_started/index
-   reference/index
-   development/index
+    :hidden:
+
+    intro/index
+    topics/index
+    reference/index

docs/intro/index.rst

@@ -0,0 +1,6 @@
+Introduction
+============
+
+.. toctree::
+
+   tutorial

docs/getting_started/quickstart.rst → docs/intro/tutorial.rst

@@ -1,5 +1,5 @@
-Quickstart
-==========
+Quickstart tutorial
+===================
 
 TL;DR
 -----
@@ -47,18 +47,18 @@ Generate PrivateKey and Exchange PublicKey
     bash examples/docker-compose/setup.sh
 
 .. note::
-   The setup.sh script will generate private key for each party, and exchange public key with peers, add peers public key to authorized_profile.json configuration. 
+   The setup.sh script will generate private key for each party, and exchange public key with peers, add peers public key to authorized_profile.json configuration.
 
 Start SCQL Service
 ------------------
 
 You could start SCQL service via `docker-compose <https://github.com/secretflow/scql/tree/main/examples/docker-compose>`_, it would deploy and start services as shown in the following figure, it contains a SCDBServer and two SCQLEngines from party ``alice``, ``bob``.
 
-.. image:: ../imgs/scql_quickstart_deploy.png
+.. image:: /imgs/scql_quickstart_deploy.png
     :alt: docker-compose deployment for quickstart example
 
 
-.. note:: 
+.. note::
     To demonstrate SCQL, we conducted the following simplified operations:
 
     1. The SCDBServer and two other SCQLEngines use the same database server but are separated by distinct database names.
@@ -81,7 +81,7 @@ SCDBServer is listening on ``http://localhost:8080``, you could send queries to
 Create database, user and tables
 --------------------------------
 
-.. code-block:: bash 
+.. code-block:: bash
 
     # use scdbclient to connect to scdbserver
     ./scdbclient prompt --host=http://localhost:8080 --usersConfFileName=examples/docker-compose/client/users.json --sync
@@ -106,15 +106,15 @@ Create database, user and tables
     | demo     |
     +----------+
     # create user "alice" with password "some_password" for party "alice"
-    # note: if you want to use a custom password, 
+    # note: if you want to use a custom password,
     # please ensure it is consistent with the one configured in `examples/docker-compose/client/users.json`
-    # 
+    #
     # NOTE: please paste the create user statement generated by scqltool
     # ./scqltool genCreateUserStmt --user alice --passwd some_password --party alice --pem examples/docker-compose/engine/alice/conf/ed25519key.pem
     root> CREATE USER `alice` PARTY_CODE 'alice' IDENTIFIED BY 'some_password' WITH '2023-08-23T20:03:34.268353853+08:00' '/oWeDbslKFQaqM6aOumnQY56i6MQKNNz84v0nkdhniXS0eBNX/q3n4IYz2EkABgKD+nkIVFtBokQqx5fr29CBw==' 'MCowBQYDK2VwAyEAzvfiNl1c1TjcvaTQBAxpG93MzHRGwuUBrPI3qf5N2XQ='
     [fetch] OK for DDL/DCL
     # create user "bob" with password "another_password" for party "bob"
-    # 
+    #
     # NOTE: please paste the create user statement generated by scqltool
     # ./scqltool genCreateUserStmt --user bob --passwd another_password --party bob --pem examples/docker-compose/engine/bob/conf/ed25519key.pem
     root> CREATE USER `bob` PARTY_CODE 'bob' IDENTIFIED BY 'another_password' WITH '2023-08-23T20:05:30.707053037+08:00' 'M5if0LyzVcBuik1C1MEHw4uGIRI6bH42CMx+O+Zpwu4IplcrQ0om0uhE7JJW910k+IGErep3vZOV8i+Icg2uBg==' 'MCowBQYDK2VwAyEAxWpev3nDG+sp9r3xMJVuqzpmRmXFQNs3e0P1Dj9DRRM='
@@ -174,7 +174,7 @@ Register SCQLEngine Endpoints
     alice> switch bob
     bob> alter user bob with ENDPOINT 'engine_bob:8003'
 
-Grant CCL 
+Grant CCL
 ---------
 
 .. code-block:: bash