Add steps API based on pre-existing run API

CHANGELOG.md

@@ -17,6 +17,9 @@ and this project adheres to
 
 ### Added
 
+- Reintroduce steps (old runs) API based on GovStack spec.
+  [#1656](https://github.com/OpenFn/lightning/issues/1656)
+
 ### Changed
 
 ### Fixed

config/test.exs

@@ -166,3 +166,5 @@ config :lightning, :github_app,
 config :lightning, LightningWeb.CollectionsController,
   default_stream_limit: 25,
   max_database_limit: 15
+
+config :lightning, LightningWeb.API.StepController, max_page_size: 10

lib/lightning/invocation.ex

@@ -259,18 +259,6 @@ defmodule Lightning.Invocation do
     Repo.all(Step)
   end
 
-  @spec list_steps_for_project_query(Lightning.Projects.Project.t()) ::
-          Ecto.Query.t()
-  def list_steps_for_project_query(%Project{id: project_id}) do
-    from(s in Step,
-      join: j in assoc(s, :job),
-      join: w in assoc(j, :workflow),
-      where: w.project_id == ^project_id,
-      order_by: [desc: s.inserted_at, desc: s.started_at],
-      preload: [job: j]
-    )
-  end
-
   @spec list_steps_for_project(Lightning.Projects.Project.t(), keyword | map) ::
           Scrivener.Page.t()
   def list_steps_for_project(%Project{} = project, params \\ %{}) do
@@ -707,4 +695,14 @@ defmodule Lightning.Invocation do
     end)
     |> Repo.transaction()
   end
+
+  defp list_steps_for_project_query(%Project{id: project_id}) do
+    from(s in Step,
+      join: j in assoc(s, :job),
+      join: w in assoc(j, :workflow),
+      where: w.project_id == ^project_id,
+      order_by: [desc: s.inserted_at, desc: s.started_at],
+      preload: [job: j]
+    )
+  end
 end

lib/lightning_web/controllers/api/step_controller.ex

@@ -0,0 +1,63 @@
+defmodule LightningWeb.API.StepController do
+  use LightningWeb, :controller
+
+  alias Lightning.Invocation
+  alias Lightning.Policies.Permissions
+  alias Lightning.Policies.ProjectUsers
+  alias Lightning.Projects
+
+  @valid_params ~w(page page_size project_id)
+  @max_page_size Application.compile_env(
+                   :lightning,
+                   LightningWeb.API.StepController
+                 )[:max_page_size] || 100
+
+  action_fallback LightningWeb.FallbackController
+
+  def index(conn, %{"project_id" => project_id} = params) do
+    with :ok <- validate_params(params),
+         :ok <- authorize_read(conn, project_id) do
+      pagination_attrs =
+        params
+        |> Map.take(["page", "page_size"])
+        |> Map.update(
+          "page_size",
+          @max_page_size,
+          &min(@max_page_size, String.to_integer(&1))
+        )
+
+      page =
+        project_id
+        |> Projects.get_project!()
+        |> Invocation.list_steps_for_project(pagination_attrs)
+
+      render(conn, "index.json", %{page: page, conn: conn})
+    end
+  end
+
+  def show(conn, %{"project_id" => project_id, "id" => id}) do
+    with :ok <- authorize_read(conn, project_id) do
+      step = Invocation.get_step_with_job!(id)
+      render(conn, "show.json", %{step: step, conn: conn})
+    end
+  end
+
+  defp validate_params(params) do
+    with [] <- Map.keys(params) -- @valid_params,
+         {_n, ""} <- Integer.parse(params["page"] || "1"),
+         {_n, ""} <- Integer.parse(params["page_size"] || "1") do
+      :ok
+    else
+      _invalid -> {:error, :bad_request}
+    end
+  end
+
+  defp authorize_read(conn, project_id) do
+    Permissions.can(
+      ProjectUsers,
+      :access_project,
+      conn.assigns.current_resource,
+      %{project_id: project_id}
+    )
+  end
+end

lib/lightning_web/controllers/api/step_json.ex

@@ -0,0 +1,41 @@
+defmodule LightningWeb.API.StepJSON do
+  @moduledoc false
+
+  def render("index.json", %{page: page}) do
+    page.entries
+    |> Enum.map(&process_instance/1)
+  end
+
+  def render("show.json", %{step: step}) do
+    process_instance(step)
+  end
+
+  defp process_instance(step) do
+    %{
+      id: step.id,
+      processRef: "#{step.job.name}:1:#{step.job.id}",
+      initTime: step.started_at,
+      state: step_state(step),
+      lastChangeTime: step.updated_at
+    }
+  end
+
+  defp step_state(step) do
+    case {step.started_at, step.finished_at, step.exit_reason} do
+      {nil, nil, _reason} ->
+        "Ready"
+
+      {_started_at, _finished_at, failed} when failed in ["cancel", "kill"] ->
+        "Terminated"
+
+      {_started_at, nil, _reason} ->
+        "Active"
+
+      {_started_at, _finished_at, "sucess"} ->
+        "Completed"
+
+      {_started_at, _finished_at, _reason} ->
+        "Failed"
+    end
+  end
+end

lib/lightning_web/router.ex

@@ -83,11 +83,10 @@ defmodule LightningWeb.Router do
     resources "/projects", API.ProjectController, only: [:index, :show] do
       resources "/jobs", API.JobController, only: [:index, :show]
       resources "/workflows", API.WorkflowsController, except: [:delete]
-      # resources "/runs", API.RunController, only: [:index, :show]
+      resources "/steps", API.StepController, only: [:index, :show]
     end
 
     resources "/jobs", API.JobController, only: [:index, :show]
-    # resources "/runs", API.RunController, only: [:index, :show]
   end
 
   ## Collections

mix.lock

@@ -26,13 +26,13 @@
   "crontab": {:hex, :crontab, "1.1.14", "233fcfdc2c74510cabdbcb800626babef414e7cb13cea11ddf62e10e16e2bf76", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "4e3b9950bc22ae8d0395ffb5f4b127a140005cba95745abf5ff9ee7e8203c6fa"},
   "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"},
   "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"},
-  "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"},
+  "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "dialyxir": {:hex, :dialyxir, "1.4.5", "ca1571ac18e0f88d4ab245f0b60fa31ff1b12cbae2b11bd25d207f865e8ae78a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b0fb08bb8107c750db5c0b324fa2df5ceaa0f9307690ee3c1f6ba5b9eb5d35c3"},
   "dotenvy": {:hex, :dotenvy, "0.8.0", "777486ad485668317c56afc53a7cbcd74f43e4e34588ba8e95a73e15a360050e", [:mix], [], "hexpm", "1f535066282388cbd109743d337ac46ff0708195780d4b5778bb83491ab1b654"},
   "earmark": {:hex, :earmark, "1.4.47", "7e7596b84fe4ebeb8751e14cbaeaf4d7a0237708f2ce43630cfd9065551f94ca", [:mix], [], "hexpm", "3e96bebea2c2d95f3b346a7ff22285bc68a99fbabdad9b655aa9c6be06c698f8"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"},
-  "ecto": {:hex, :ecto, "3.12.5", "4a312960ce612e17337e7cefcf9be45b95a3be6b36b6f94dfb3d8c361d631866", [], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6eb18e80bef8bb57e17f5a7f068a1719fbda384d40fc37acb8eb8aeca493b6ea"},
+  "ecto": {:hex, :ecto, "3.12.5", "4a312960ce612e17337e7cefcf9be45b95a3be6b36b6f94dfb3d8c361d631866", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6eb18e80bef8bb57e17f5a7f068a1719fbda384d40fc37acb8eb8aeca493b6ea"},
   "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
   "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.8.2", "79350a53246ac5ec27326d208496aebceb77fa82a91744f66a9154560f0759d3", [:mix], [{:ecto_sql, "~> 3.7", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, "> 0.16.0 and < 0.20.0", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1 or ~> 4.0.0", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "6149c1c4a5ba6602a76cb09ee7a269eb60dab9694a1dbbb797f032555212de75"},
   "ecto_sql": {:hex, :ecto_sql, "3.12.1", "c0d0d60e85d9ff4631f12bafa454bc392ce8b9ec83531a412c12a0d415a3a4d0", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aff5b958a899762c5f09028c847569f7dfb9cc9d63bdb8133bff8a5546de6bf5"},