From 1c91c895c4337910ac7dc5f94c50037c68f51bde Mon Sep 17 00:00:00 2001
From: Gonzalo Rafuls <grafuls@redhat.com>
Date: Fri, 7 Feb 2020 03:15:14 -0500
Subject: [PATCH] feat: pbench-server REST API

Added pbench-server to be executed by systemctl.
Additional changes required on RPM build spec.

feat: agent side of transfer REST

feat: moved util-scripts to own utils dir

final prototype for move_results

feat: moved read in chunks and post to copy_results_tb

reverted util-scripts changes

fixed revert

feat: refactored pbench-make-result-tb to python

feat: converted copy-results-tb

added get on pbench-server for check sum

fix: formatting on logger

feat: added first test for pbench-server

added test for post

added tests plus mod gitignore

feat: fixed and moved tests around for pbench-server REST

feat: fixes to tests plus metadata.log fixure cleanup

general refactoring

feat: reafactored and added tests

feat: gral tests refactoring

removed config invocation for agent

added host_info entrypoint test

feat: tests refactoring

feat: added pytests to travis

fix: travisyml conflicts

fix: removed new line on init

fix: travis yml python path

fix: travis yml python script

fix: added missing libraries for travis

fix: added werkzeug library to travis install

fix: moved python rpm package to be systems one

fix: added system site packages for travis virtualenv

feat: added tests and pytest.ini

fix: log formatting plus added tests

fix: fixes to TCRT tests

Addressing review comments

Moved agent utils to lib/pbench and config ammendments

fix to travis.yml and tests references

general refactoring

Added libs to pythonpath for travis

removed unused import

fix to travis configtools import

Added _PBENCH_*_CONFIG env variables to travis

Fix for fixtures paths

fix on test fixture variables

fix for travis env yaml

fix for server tests

Refactored config on server side to accomodate to pytest

moved logger to init

fixed configtools refactor

fix for config fixture install_dir

Fix for test copy results

fix to server tests

Fixes to server tests

fixed flake8 issues

black lint fixes

made config methods

blacked

Fixes to config

Added argparse for move_results plus removed check entrypoint for server

Addressing comments from PR

Fixes to agent tests

fix for test copy result

Fixes to test move results fixtures

fix to tox pythonpath

fix for pytest execution

Fixed requirements/removed pytest-flask

running pytest via python

Fixes to pythonpath for tox

fixes to tox pythonpath

moved pythonpath to envdir in tox

Added build-system to pytproject toml

added requires to build-system

Fix configtools import

fix configtools import

moved rpm package to pipy

removed sitepackages from tox

Fixes to fixtures

removed rpy-py-installer

Removed rpm for host info

modified env from VIRTUAL_ENV to PYTHONPATH

Fix for server test setup

removed fixtures scope plus redundant packages on test-requirements

Added fixture decorator

Added pytest-flask package

fixes to server conftest

blacked

fixes to server tests

Final fixes to tests

blacked

fixes to add-metalog-option

additional fixes
---
 .gitignore                                    |   2 +
 .travis.yml                                   |   2 +
 agent/config/pbench-agent-default.cfg         |   9 +-
 agent/lib/pbench/__init__.py                  |   7 +
 agent/lib/pbench/add_metalog_option.py        |  39 ++++
 agent/lib/pbench/config.py                    |  16 ++
 agent/lib/pbench/copy_result_tb.py            |  71 ++++++
 agent/lib/pbench/make_result_tb.py            | 124 ++++++++++
 agent/lib/pbench/move_results.py              | 142 ++++++++++++
 agent/lib/pbench/tests/common.py              |  19 ++
 .../tests/fixtures/config/pbench-agent.cfg    |  10 +
 .../tests/fixtures/copy_result_tb/log.tar.xz  | Bin 0 -> 640 bytes
 .../copy_result_tb/log.tar.xz.md5.check       |   1 +
 .../tests/fixtures/make_result_tb/fixture.log |   1 +
 .../fixtures/make_result_tb/metadata.log      |   8 +
 .../fixtures/move_results/logs/fixture.log    |   1 +
 .../fixtures/move_results/logs/metadata.log   |   9 +
 agent/lib/pbench/tests/test_copy_result_tb.py |  82 +++++++
 agent/lib/pbench/tests/test_make_result_tb.py |  95 ++++++++
 agent/lib/pbench/tests/test_move_results.py   |  32 +++
 doc/CONTRIBUTING.md                           |   2 +
 pbench/common/configtools.py                  |  17 +-
 pyproject.toml                                |   7 +
 pytest.ini                                    |   3 +
 requirements.txt                              |   4 +
 run-unittests                                 |   2 +-
 server/lib/config/pbench-server-default.cfg   |   7 +
 server/lib/config/pbench-server.cfg.example   |   7 +
 server/lib/pbench_server/__init__.py          |   0
 server/lib/pbench_server/config.py            |  31 +++
 server/lib/pbench_server/pbench_server.py     | 124 ++++++++++
 server/lib/pbench_server/tests/common.py      |   7 +
 server/lib/pbench_server/tests/conftest.py    |  36 +++
 .../tests/fixtures/config/pbench-server.cfg   | 212 ++++++++++++++++++
 .../tests/fixtures/upload/log.tar.xz          | Bin 0 -> 640 bytes
 .../fixtures/upload/log.tar.xz.md5.check      |   1 +
 .../lib/pbench_server/tests/test_requests.py  |  55 +++++
 server/lib/systemd/pbench-server.service      |  25 +++
 test-requirements.txt                         |   1 +
 tox.ini                                       |  24 +-
 40 files changed, 1222 insertions(+), 13 deletions(-)
 create mode 100644 agent/lib/pbench/__init__.py
 create mode 100755 agent/lib/pbench/add_metalog_option.py
 create mode 100644 agent/lib/pbench/config.py
 create mode 100644 agent/lib/pbench/copy_result_tb.py
 create mode 100644 agent/lib/pbench/make_result_tb.py
 create mode 100755 agent/lib/pbench/move_results.py
 create mode 100644 agent/lib/pbench/tests/common.py
 create mode 100644 agent/lib/pbench/tests/fixtures/config/pbench-agent.cfg
 create mode 100644 agent/lib/pbench/tests/fixtures/copy_result_tb/log.tar.xz
 create mode 100644 agent/lib/pbench/tests/fixtures/copy_result_tb/log.tar.xz.md5.check
 create mode 100644 agent/lib/pbench/tests/fixtures/make_result_tb/fixture.log
 create mode 100644 agent/lib/pbench/tests/fixtures/make_result_tb/metadata.log
 create mode 100644 agent/lib/pbench/tests/fixtures/move_results/logs/fixture.log
 create mode 100644 agent/lib/pbench/tests/fixtures/move_results/logs/metadata.log
 create mode 100644 agent/lib/pbench/tests/test_copy_result_tb.py
 create mode 100644 agent/lib/pbench/tests/test_make_result_tb.py
 create mode 100644 agent/lib/pbench/tests/test_move_results.py
 create mode 100644 pytest.ini
 create mode 100644 server/lib/pbench_server/__init__.py
 create mode 100644 server/lib/pbench_server/config.py
 create mode 100644 server/lib/pbench_server/pbench_server.py
 create mode 100644 server/lib/pbench_server/tests/common.py
 create mode 100644 server/lib/pbench_server/tests/conftest.py
 create mode 100644 server/lib/pbench_server/tests/fixtures/config/pbench-server.cfg
 create mode 100644 server/lib/pbench_server/tests/fixtures/upload/log.tar.xz
 create mode 100644 server/lib/pbench_server/tests/fixtures/upload/log.tar.xz.md5.check
 create mode 100644 server/lib/pbench_server/tests/test_requests.py
 create mode 100644 server/lib/systemd/pbench-server.service

diff --git a/.gitignore b/.gitignore
index 5d03c09ff2..93687d53a6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,5 @@ dist
 pip-wheel-metadata
 AUTHORS
 ChangeLog
+.idea/
+.pytest_cache/
diff --git a/.travis.yml b/.travis.yml
index 2ff1aba9c8..1077077249 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,8 @@ env:
   - TOXENV=lint
   - TOXENV=agent
   - TOXENV=server
+  - TOXENV=pytest_agent
+  - TOXENV=pytest_server
   - TOXENV=py3
 before_install:
  - sudo apt-get update
diff --git a/agent/config/pbench-agent-default.cfg b/agent/config/pbench-agent-default.cfg
index 608f9b63de..f20c860e53 100644
--- a/agent/config/pbench-agent-default.cfg
+++ b/agent/config/pbench-agent-default.cfg
@@ -1,15 +1,22 @@
 [DEFAULT]
 version = 002
+pbench_web_server = pbench.example.com
 
 [results]
 user = pbench
 host_path = http://%(pbench_result_redirector)s/pbench-archive-host
 webserver = %(pbench_web_server)s
-host_info_url = http://%(webserver)s/pbench-results-host-info.versioned/pbench-results-host-info.URL002
+host_info_uri = pbench-results-host-info.versioned/pbench-results-host-info.URL002
+host_info_url = http://%(webserver)s/%(host_info_uri)s
 dir = /srv/pbench/public_html/incoming
 scp_opts = -o StrictHostKeyChecking=no
 ssh_opts = -o StrictHostKeyChecking=no
 
+# REST API entrypoint
+api_version = 1
+rest_endpoint = api/v%(api_version)s
+server_rest_url = http://%(webserver)s/%(rest_endpoint)s
+
 [pbench-agent]
 install-dir = %(pbench_install_dir)s
 pbench_user = pbench
diff --git a/agent/lib/pbench/__init__.py b/agent/lib/pbench/__init__.py
new file mode 100644
index 0000000000..6b7d91432e
--- /dev/null
+++ b/agent/lib/pbench/__init__.py
@@ -0,0 +1,7 @@
+import logging
+import sys
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.StreamHandler(sys.stdout))
+logger.propagate = False
+logging.basicConfig(level=logging.INFO, format="%(message)s")
diff --git a/agent/lib/pbench/add_metalog_option.py b/agent/lib/pbench/add_metalog_option.py
new file mode 100755
index 0000000000..a404748688
--- /dev/null
+++ b/agent/lib/pbench/add_metalog_option.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+# Usage: pbench-add-metalog-option <metadata log file> <section> <option>
+
+# Add an option to a section of the metadata.log file.
+# E.g. using an 'iterations' arg for the option
+#
+# iterations: 1-iter, 2-iter, 3-iter
+#
+# where the iterations are in the <iterations.file>, one iteration per line.
+import errno
+import sys
+from configparser import ConfigParser, NoSectionError
+
+
+def add_metalog_option(log_file, section, option, value):
+    config = ConfigParser()
+
+    try:
+        config.read(log_file)
+    except OSError as ex:
+        if ex.errno == errno.ENOENT:
+            raise Exception(f"Log file does not exist: {log_file}")
+        raise
+
+    try:
+        config.set(section, option, value)
+    except NoSectionError:
+        config.add_section(section)
+        config.set(section, option, value)
+    config.write(open(log_file, "w"))
+
+
+if __name__ == "__main__":
+    _log_file = sys.argv[1]
+    _section = sys.argv[2]
+    _option = sys.argv[3]
+    _value = sys.argv[4]
+    add_metalog_option(_log_file, _section, _option, _value)
diff --git a/agent/lib/pbench/config.py b/agent/lib/pbench/config.py
new file mode 100644
index 0000000000..e64e2d5f15
--- /dev/null
+++ b/agent/lib/pbench/config.py
@@ -0,0 +1,16 @@
+from pbench.common import configtools
+
+
+class AgentConfig:
+    def __init__(self):
+        opts, _ = configtools.parse_args()
+        pbench_config, _ = configtools.init(opts, "_PBENCH_AGENT_CONFIG")
+
+        self.agent = pbench_config["pbench-agent"]
+        self.results = pbench_config["results"]
+
+    def get_agent(self):
+        return self.agent
+
+    def get_results(self):
+        return self.results
diff --git a/agent/lib/pbench/copy_result_tb.py b/agent/lib/pbench/copy_result_tb.py
new file mode 100644
index 0000000000..b9b84a0c56
--- /dev/null
+++ b/agent/lib/pbench/copy_result_tb.py
@@ -0,0 +1,71 @@
+import os
+import sys
+from pathlib import Path
+
+import requests
+
+from agent.lib.pbench import logger
+from agent.lib.pbench.config import AgentConfig
+from werkzeug.utils import secure_filename
+
+
+class CopyResultTb:
+    def __init__(self, tarball, path, _logger=None):
+        config = AgentConfig()
+        server_rest_url = config.results.get("server_rest_url")
+
+        self.tarball = tarball
+        self.path = path
+        self.chunk_size = 4096
+        self.logger = logger if not _logger else _logger
+        self.upload_url = f"{server_rest_url}/upload"
+        self.controller_dir = os.path.dirname(self.tarball)
+        self.controller = os.path.basename(self.controller_dir)
+
+    def read_in_chunks(self, file_object):
+        while True:
+            data = file_object.read(self.chunk_size)
+            if not data:
+                break
+            yield data
+
+    def post_file(self, file):
+        filename = secure_filename(os.path.join(self.path, file))
+        with open(f"{os.path.join(self.path, file)}.md5.check", "r") as _check:
+            md5sum = _check.read()
+        headers = {"filename": filename, "md5sum": md5sum}
+        content_path = os.path.abspath(file)
+        with open(content_path, "rb") as f:
+            try:
+                response = requests.post(
+                    self.upload_url, data=self.read_in_chunks(f), headers=headers
+                )
+                self.logger.info("File uploaded successfully")
+            except Exception:
+                self.logger.exception("There was something wrong with your request")
+                sys.exit(1)
+        if not response.status_code == 200:
+            self.logger.error("There was something wrong with your request")
+            self.logger.error("Error code: %s" % response.status_code)
+            sys.exit(1)
+
+    def copy_result_tb(self):
+        if not os.path.exists(self.tarball):
+            self.logger.error("tarball does not exist, %s" % self.tarball)
+            sys.exit(1)
+        if not os.path.exists(f"{self.tarball}.md5.check"):
+            self.logger.error(
+                "tarball's .md5.check does not exist, %s.md5.check" % self.tarball
+            )
+            sys.exit(1)
+
+        files = [file for file in Path(self.controller_dir).iterdir() if file.is_file()]
+        file_count = len(files)
+        if file_count != 2:
+            self.logger.error(
+                "(internal): unexpected file count, %s, associated with tarball, %s"
+                % (file_count, self.tarball)
+            )
+            sys.exit(1)
+
+        self.post_file(self.tarball)
diff --git a/agent/lib/pbench/make_result_tb.py b/agent/lib/pbench/make_result_tb.py
new file mode 100644
index 0000000000..aa04f01b64
--- /dev/null
+++ b/agent/lib/pbench/make_result_tb.py
@@ -0,0 +1,124 @@
+import hashlib
+import os
+import sys
+import tarfile
+import datetime
+
+from agent.lib.pbench import logger
+from agent.lib.pbench.add_metalog_option import add_metalog_option
+from agent.lib.pbench.config import AgentConfig
+from pathlib import Path
+from pbench.common import configtools
+
+
+class MakeResultTb:
+    def __init__(self, result_dir, target_dir, user, prefix, _logger=None):
+        self.result_dir = result_dir
+        self.target_dir = target_dir
+        self.user = user
+        self.prefix = prefix
+        self.logger = logger if not _logger else _logger
+
+    def make_result_tb(self):
+        config = AgentConfig()
+
+        if not os.path.exists(self.result_dir):
+            self.logger.error("Invalid result directory provided: %s" % self.result_dir)
+            sys.exit(1)
+
+        if not os.path.exists(self.target_dir):
+            self.logger.error("Invalid target directory provided: %s" % self.target_dir)
+            sys.exit(1)
+
+        full_result_dir = os.path.realpath(self.result_dir)
+        pbench_run_name = os.path.basename(self.result_dir)
+        if os.path.exists(f"{full_result_dir}.copied"):
+            self.logger.debug("Already copied %s" % self.result_dir)
+            sys.exit(0)
+
+        if os.path.exists(f"{full_result_dir}/.running"):
+            self.logger.debug(
+                "The benchmark is still running in %s - skipping" % pbench_run_name
+            )
+            self.logger.debug(
+                "If that is not true, rmdir %s/.running, and try again"
+                % pbench_run_name
+            )
+            sys.exit(0)
+
+        md_log = f"{full_result_dir}/metadata.log"
+        if not os.path.exists(md_log):
+            self.logger.debug(
+                "The %s/metadata.log file seems to be missing", pbench_run_name
+            )
+            sys.exit(0)
+
+        opts, _ = configtools.parse_args()
+        opts.filename = md_log
+        conf_md, _ = configtools.init(opts, None)
+        md_config = conf_md["pbench"]
+        res_name = md_config.get("name")
+        pbench_run = config.agent.get("pbench_run")
+        if res_name != pbench_run_name:
+            self.logger.warning(
+                "The run in directory %s/%s "
+                "has an unexpected metadata name, '%s' - skipping"
+                % (pbench_run, pbench_run_name, res_name)
+            )
+            sys.exit(1)
+
+        if self.user:
+            add_metalog_option(md_log, "run", "user", self.user)
+
+        if self.prefix:
+            add_metalog_option(md_log, "run", "prefix", self.prefix)
+
+        result_size = sum(
+            file.stat().st_size for file in Path(full_result_dir).rglob("*")
+        )
+        self.logger.debug(
+            "preparing to tar up %s bytes of data from %s"
+            % (result_size, full_result_dir)
+        )
+        add_metalog_option(md_log, "run", "raw_size", f"{result_size}")
+
+        timestamp = datetime.datetime.isoformat(datetime.datetime.now())
+        add_metalog_option(
+            md_log, "pbench", "tar-ball-creation-timestamp", f"{timestamp}"
+        )
+
+        tarball = os.path.join(self.target_dir, f"{pbench_run_name}.tar.xz")
+        files = [os.path.realpath(file) for file in Path(full_result_dir).rglob("*")]
+        try:
+            with tarfile.open(tarball, mode="x:xz") as tar:
+                for f in files:
+                    tar.add(f)
+        except tarfile.TarError:
+            self.logger.error(
+                "tar ball creation failed for %s, skipping" % self.result_dir
+            )
+            if os.path.exists(tarball):
+                os.remove(tarball)
+            sys.exit(1)
+
+        self.make_md5sum(tarball)
+
+        return tarball
+
+    def make_md5sum(self, tarball):
+        tarball_md5 = f"{tarball}.md5.check"
+        try:
+            hash_md5 = hashlib.md5()
+            with open(tarball, "rb") as tar:
+                for chunk in iter(lambda: tar.read(4096), b""):
+                    hash_md5.update(chunk)
+
+            with open(tarball_md5, "w") as md5:
+                md5.write(hash_md5.hexdigest())
+        except Exception:
+            self.logger.error("md5sum failed for %s, skipping" % tarball)
+            if os.path.exists(tarball):
+                os.remove(tarball)
+            if os.path.exists(tarball_md5):
+                os.remove(tarball_md5)
+            sys.exit(1)
diff --git a/agent/lib/pbench/move_results.py b/agent/lib/pbench/move_results.py
new file mode 100755
index 0000000000..c5a927ac9e
--- /dev/null
+++ b/agent/lib/pbench/move_results.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import sys
+import requests
+
+from agent.lib.pbench import logger
+from agent.lib.pbench.config import AgentConfig
+from agent.lib.pbench.make_result_tb import MakeResultTb
+from agent.lib.pbench.copy_result_tb import CopyResultTb
+
+
+def move_results(_user, _prefix, _show_server):
+    config = AgentConfig()
+    config_agent = config.get_agent()
+    config_results = config.get_results()
+    if not _user:
+        _user = config_agent.get("pbench_user")
+    pbench_run = config_agent.get("pbench_run")
+    if not pbench_run:
+        logger.error("No pbench_run configured under pbench-agent section")
+        sys.exit(1)
+    if not os.path.exists(pbench_run):
+        logger.error("pbench local results directory does not exist: %s", pbench_run)
+        sys.exit(1)
+
+    controller = os.environ.get("full_hostname")
+    if not controller:
+        logger.error("Missing controller name (should be 'hostname -f' value)")
+        sys.exit(1)
+
+    results_webserver = config_results.get("webserver")
+    if not results_webserver:
+        logger.error(
+            "No web server host configured from which we can fetch the FQDN of the host to which we copy/move results"
+        )
+        logger.debug("'webserver' variable in 'results' section not set")
+
+    host_info_url = config_results.get("host_info_url")
+    server_rest_url = config_results.get("server_rest_url")
+    response = requests.get(f"{server_rest_url}/host_info")
+
+    if response.status_code not in [200, 201]:
+        logger.error("Unable to determine results host info from %s", host_info_url)
+        sys.exit(1)
+    if response.text.startswith("MESSAGE"):
+        message = response.text.split("===")[1]
+        logger.info("*** Message from sysadmins of %s:", results_webserver)
+        logger.info("***\n*** %s", message)
+        logger.info("***\n*** No local actions taken.")
+        sys.exit(1)
+    results_path_prefix = response.text.split(":")[1]
+    if not results_path_prefix:
+        logger.error(
+            "fetch results host info did not contain a path prefix: %s", response.text
+        )
+        sys.exit(1)
+
+    runs_copied = 0
+    failures = 0
+
+    pbench_tmp = os.environ.get("pbench_tmp")
+    temp_dir = os.path.join(
+        pbench_tmp, ".".join([os.path.basename(__file__), str(os.getpid())]), controller
+    )
+    try:
+        os.makedirs(temp_dir)
+    except OSError:
+        logger.error("Failed to create temporary directory")
+        sys.exit(1)
+
+    dirs = [
+        _dir
+        for _dir in next(os.walk(pbench_run))[1]
+        if not _dir.startswith("tools-") and not _dir.startswith("tmp")
+    ]
+
+    for _dir in dirs:
+        result_dir = os.path.join(pbench_run, _dir)
+        mrt = MakeResultTb(result_dir, temp_dir, _user, _prefix)
+        result_tb_name = mrt.make_result_tb()
+        if result_tb_name:
+            crt = CopyResultTb(result_tb_name, results_path_prefix)
+            copy_result = crt.copy_result_tb()
+            try:
+                os.remove(result_tb_name)
+                os.remove(f"{result_tb_name}.md5.check")
+            except OSError:
+                logger.error(
+                    "rm failed to remove %s and its .md5.check file", result_tb_name
+                )
+                sys.exit(1)
+            if not copy_result:
+                failures += 1
+                continue
+
+            try:
+                os.remove(result_dir)
+            except OSError:
+                logger.error(
+                    "rm failed to remove the %s directory hierarchy", result_dir
+                )
+                sys.exit(1)
+
+            runs_copied += 1
+
+    if runs_copied + failures > 0:
+        logger.debug(
+            "successfully moved %s runs, encountered %s failures", runs_copied, failures
+        )
+
+    return failures
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Move pbench results to pbench server")
+    parser.add_argument(
+        "-u",
+        "--user",
+        dest="user",
+        type=str,
+        default="",
+        help="Specify username for server upload",
+    )
+    parser.add_argument(
+        "-p",
+        "--prefix",
+        dest="prefix",
+        type=str,
+        default="",
+        help="Specify a prefix for server upload",
+    )
+    parser.add_argument(
+        "-S",
+        "--show-server",
+        dest="show_server",
+        action="store_false",
+        help="Show server",
+    )
+
+    args = parser.parse_args()
+    move_results(args.user, args.prefix, args.show_server)
diff --git a/agent/lib/pbench/tests/common.py b/agent/lib/pbench/tests/common.py
new file mode 100644
index 0000000000..ad6065d97b
--- /dev/null
+++ b/agent/lib/pbench/tests/common.py
@@ -0,0 +1,19 @@
+import datetime
+import tempfile
+
+FAKE_TIME = datetime.datetime(2020, 2, 2, 22, 0, 0)
+target_dir = tempfile.mkdtemp()
+tarball = "agent/lib/pbench/tests/fixtures/copy_result_tb/log.tar.xz"
+bad_tarball = "nothing.tar.xz"
+
+MRT_DIR = "agent/lib/pbench/tests/fixtures/make_result_tb"
+
+
+class MockDatetime(datetime.datetime):
+    @classmethod
+    def now(cls, **kwargs):
+        return FAKE_TIME
+
+
+def mock_agent_config(*args, **kwargs):
+    return {"pbench_run": "agent/lib/pbench/tests/fixtures/move_results"}
diff --git a/agent/lib/pbench/tests/fixtures/config/pbench-agent.cfg b/agent/lib/pbench/tests/fixtures/config/pbench-agent.cfg
new file mode 100644
index 0000000000..27e68ea039
--- /dev/null
+++ b/agent/lib/pbench/tests/fixtures/config/pbench-agent.cfg
@@ -0,0 +1,10 @@
+[DEFAULT]
+pbench_install_dir = ../../../../..
+pbench_results_redirector = pbench.results.example.com
+pbench_web_server = pbench.example.com
+
+version = 002
+
+[config]
+path = %(pbench_install_dir)s/config
+files = pbench-agent-default.cfg
\ No newline at end of file
diff --git a/agent/lib/pbench/tests/fixtures/copy_result_tb/log.tar.xz b/agent/lib/pbench/tests/fixtures/copy_result_tb/log.tar.xz
new file mode 100644
index 0000000000000000000000000000000000000000..e3c9e1f6bc8479122090a98322f5269c609d9162
GIT binary patch
literal 640
zcmV-`0)PD<iwFRz?@wL;1MQW|j?*v{hP(DtcsC&7)R){vS+axG?5e7A6Q_-u#KCqt
zz|&)=FhfU01GAVB()qH8buK=3{2#|Eb5dsMly!F5v|rt1lwJ1nsJ4DEb#Ci+zwA+`
z_lsvhJ!u+;H_H<G`+J6^Y{5jz1y2%2y`A$oNfwwsr`W$TStkv~i@lM})Y(7AJ>~yT
z`KWf7rI?B`p(W+G)0ONF+B*DMxjSu+pX4C-%B&g7@=b}~w?m^pV7z8Y&hl7L%A=GQ
z#Rk`hYK-xlyKdx`zAZ7vgY1=^^@VR3jl#ojUVFp91@>?HzgJGy(#dCD08jKk6GEK(
zKT9)_iO~N^2L1mctjC=is>23_X2mOfdGg3srX8KPpTo%H^+M)hFU=8EdT83ccMtWE
z1Pu{}yimeKoU)wIgm7`|FBi^A!I2qz;?-tbJ@^P3C6uszRuoE7o}kt#Bc1nI?_}>V
zL;hsQPIeusjG8^fwf;(RvLELPoD#|jXScK{6H&%llqRX51ww15b!Rsy++<-o*^LKz
zYP(LMz0p%huT>-cE`P-?A41M);e(JzxvjU$@Hq%+2fy5t>{bHx<2CE3Of%bzs<f$z
z9-UpC$CyZOZ;bk~^8Y(Dw)K?>yxr*PY%tR}Z{xB+aG&~XwXK)*Ba?6J-PP5jwV8%Z
zI7(-<-=w+NauiP7ezTEL^+CF*(me_t<F-EQ`Y}f$&r+U?j3PDE*W-98iYO8O2a?5%
zqEO^sWXQ<7t!=*t73Fd47x<uzCEGmcWcyRIY|h`_Yt?p2jR?JVYXJZN0000000000
a00000000000002Mi}3^c!)qS^C;$Myd_{2p

literal 0
HcmV?d00001

diff --git a/agent/lib/pbench/tests/fixtures/copy_result_tb/log.tar.xz.md5.check b/agent/lib/pbench/tests/fixtures/copy_result_tb/log.tar.xz.md5.check
new file mode 100644
index 0000000000..54c63eea74
--- /dev/null
+++ b/agent/lib/pbench/tests/fixtures/copy_result_tb/log.tar.xz.md5.check
@@ -0,0 +1 @@
+9d5a479f6f75fa9b3bab27ef79ad5b29
\ No newline at end of file
diff --git a/agent/lib/pbench/tests/fixtures/make_result_tb/fixture.log b/agent/lib/pbench/tests/fixtures/make_result_tb/fixture.log
new file mode 100644
index 0000000000..b06c22a566
--- /dev/null
+++ b/agent/lib/pbench/tests/fixtures/make_result_tb/fixture.log
@@ -0,0 +1 @@
+Feb 16 03:50:02 localhost NetworkManager[1178]: <info>  [1581843002.6299] dhcp4 (wlp2s0):   nameserver '10.0.0.1'
\ No newline at end of file
diff --git a/agent/lib/pbench/tests/fixtures/make_result_tb/metadata.log b/agent/lib/pbench/tests/fixtures/make_result_tb/metadata.log
new file mode 100644
index 0000000000..ae42236bce
--- /dev/null
+++ b/agent/lib/pbench/tests/fixtures/make_result_tb/metadata.log
@@ -0,0 +1,8 @@
+[pbench]
+name = make_result_tb
+tar-ball-creation-timestamp = 2020-02-02T22:00:00
+
+[run]
+user = pbench
+raw_size = 231
+
diff --git a/agent/lib/pbench/tests/fixtures/move_results/logs/fixture.log b/agent/lib/pbench/tests/fixtures/move_results/logs/fixture.log
new file mode 100644
index 0000000000..b06c22a566
--- /dev/null
+++ b/agent/lib/pbench/tests/fixtures/move_results/logs/fixture.log
@@ -0,0 +1 @@
+Feb 16 03:50:02 localhost NetworkManager[1178]: <info>  [1581843002.6299] dhcp4 (wlp2s0):   nameserver '10.0.0.1'
\ No newline at end of file
diff --git a/agent/lib/pbench/tests/fixtures/move_results/logs/metadata.log b/agent/lib/pbench/tests/fixtures/move_results/logs/metadata.log
new file mode 100644
index 0000000000..9af0331067
--- /dev/null
+++ b/agent/lib/pbench/tests/fixtures/move_results/logs/metadata.log
@@ -0,0 +1,9 @@
+[pbench]
+name = logs
+tar-ball-creation-timestamp = 2020-02-02T22:00:00
+
+[run]
+user = pbench
+raw_size = 281
+prefix = /srv/pbench/pbench-move-results-receive/fs-version
+
diff --git a/agent/lib/pbench/tests/test_copy_result_tb.py b/agent/lib/pbench/tests/test_copy_result_tb.py
new file mode 100644
index 0000000000..392bb97a6d
--- /dev/null
+++ b/agent/lib/pbench/tests/test_copy_result_tb.py
@@ -0,0 +1,82 @@
+import logging
+import os
+from tempfile import NamedTemporaryFile
+
+import responses
+
+from agent.lib.pbench.copy_result_tb import CopyResultTb
+from agent.lib.pbench.tests.common import tarball, bad_tarball
+
+
+class TestCopyResults:
+    logger = logging.getLogger()
+
+    @staticmethod
+    @responses.activate
+    def test_copy_tar():
+        responses.add(
+            responses.POST, "http://pbench.example.com/api/v1/upload", status=200
+        )
+        try:
+            crt = CopyResultTb(tarball, "")
+            crt.copy_result_tb()
+        except SystemExit:
+            assert False
+        assert True
+
+    @responses.activate
+    def test_bad_tar(self, caplog):
+        caplog.set_level(logging.ERROR, logger=self.logger.name)
+        responses.add(
+            responses.POST, "http://pbench.example.com/api/v1/upload", status=200
+        )
+        expected_error_message = "tarball does not exist, %s" % bad_tarball
+        try:
+            crt = CopyResultTb(bad_tarball, "", self.logger)
+            crt.copy_result_tb()
+        except SystemExit:
+            assert caplog.records
+            assert caplog.records[0].msg == expected_error_message
+        else:
+            assert False
+
+    @responses.activate
+    def test_missing_md5(self, caplog):
+        caplog.set_level(logging.ERROR, logger=self.logger.name)
+        responses.add(
+            responses.POST, "http://pbench.example.com/api/v1/upload", status=200
+        )
+        with NamedTemporaryFile(suffix=".tar.xz") as missing_md5_tar:
+            expected_error_message = (
+                "tarball's .md5.check does not exist, %s.md5.check"
+                % missing_md5_tar.name
+            )
+            try:
+                crt = CopyResultTb(missing_md5_tar.name, "", self.logger)
+                crt.copy_result_tb()
+            except SystemExit:
+                assert caplog.records
+                assert caplog.records[0].msg == expected_error_message
+            else:
+                assert False
+
+    @responses.activate
+    def test_multiple_files(self, caplog):
+        caplog.set_level(logging.ERROR, logger=self.logger.name)
+        responses.add(
+            responses.POST, "http://pbench.example.com/api/v1/upload", status=200
+        )
+        base_dir = os.path.dirname(tarball)
+        with NamedTemporaryFile(suffix=".add", dir=base_dir):
+            expected_error_message = (
+                "(internal): unexpected file count, 3, associated with tarball, %s"
+                % tarball
+            )
+            try:
+                crt = CopyResultTb(tarball, "", self.logger)
+                crt.copy_result_tb()
+            except SystemExit:
+                assert caplog.records
+                assert caplog.records[0].msg == expected_error_message
+            else:
+                assert False
diff --git a/agent/lib/pbench/tests/test_make_result_tb.py b/agent/lib/pbench/tests/test_make_result_tb.py
new file mode 100644
index 0000000000..e723a40c99
--- /dev/null
+++ b/agent/lib/pbench/tests/test_make_result_tb.py
@@ -0,0 +1,95 @@
+import datetime
+import logging
+import os
+
+import pytest
+
+from agent.lib.pbench.make_result_tb import MakeResultTb
+from agent.lib.pbench.tests.common import MockDatetime, target_dir, MRT_DIR
+
+
+class TestMakeResultTb:
+    logger = logging.getLogger()
+
+    @pytest.fixture(autouse=True)
+    def teardown(self):
+        if os.path.exists(f"{os.path.realpath(MRT_DIR)}.copied"):
+            os.remove(f"{os.path.realpath(MRT_DIR)}.copied")
+        if os.path.exists(f"{os.path.realpath(MRT_DIR)}/.running"):
+            os.remove(f"{os.path.realpath(MRT_DIR)}/.running")
+
+    @pytest.mark.parametrize("result_dir", ("bad/bad/result/dir", ""))
+    def test_bad_result_dir(self, result_dir, caplog):
+        caplog.set_level(logging.ERROR, logger=self.logger.name)
+        expected_error_message = "Invalid result directory provided: %s" % result_dir
+        try:
+            mrt = MakeResultTb(
+                result_dir, target_dir, "pbench", "", _logger=self.logger
+            )
+            mrt.make_result_tb()
+        except SystemExit:
+            assert caplog.records
+            assert caplog.records[0].msg == expected_error_message
+        else:
+            assert False
+
+    @pytest.mark.parametrize("target_dir", ("bad/bad/result/dir", ""))
+    def test_bad_target_dir(self, target_dir, caplog):
+        logger = logging.getLogger()
+        caplog.set_level(logging.ERROR, logger=self.logger.name)
+        expected_error_message = "Invalid target directory provided: %s" % target_dir
+        try:
+            mrt = MakeResultTb(MRT_DIR, target_dir, "pbench", "", _logger=logger)
+            mrt.make_result_tb()
+        except SystemExit:
+            assert caplog.records
+            assert caplog.records[0].msg == expected_error_message
+        else:
+            assert False
+
+    def test_already_copied(self, caplog):
+        caplog.set_level(logging.DEBUG, logger=self.logger.name)
+        first_expected_error_message = (
+            f"The benchmark is still running in {os.path.basename(MRT_DIR)} - skipping"
+        )
+        second_expected_error_message = f"If that is not true, rmdir {os.path.basename(MRT_DIR)}/.running, and try again"
+        full_result_dir = os.path.realpath(MRT_DIR)
+        with open(f"{full_result_dir}/.running", "x"):
+            try:
+                mrt = MakeResultTb(
+                    MRT_DIR, target_dir, "pbench", "", _logger=self.logger
+                )
+                mrt.make_result_tb()
+            except SystemExit:
+                assert caplog.records
+                assert caplog.records[0].msg == first_expected_error_message
+                assert caplog.records[1].msg == second_expected_error_message
+            else:
+                assert False
+
+    def test_running(self, caplog):
+        caplog.set_level(logging.DEBUG, logger=self.logger.name)
+        expected_error_message = "Already copied %s" % MRT_DIR
+        full_result_dir = os.path.realpath(MRT_DIR)
+        with open(f"{full_result_dir}.copied", "x"):
+            try:
+                mrt = MakeResultTb(
+                    MRT_DIR, target_dir, "pbench", "", _logger=self.logger
+                )
+                mrt.make_result_tb()
+            except SystemExit:
+                assert caplog.records
+                assert caplog.records[0].msg == expected_error_message
+            else:
+                assert False
+
+    @staticmethod
+    def test_make_tb(monkeypatch):
+        monkeypatch.setattr(datetime, "datetime", MockDatetime)
+        try:
+            mrt = MakeResultTb(MRT_DIR, target_dir, "pbench", "")
+            tarball = mrt.make_result_tb()
+        except SystemExit:
+            assert False
+        assert tarball == os.path.join(target_dir, "make_result_tb.tar.xz")
+        assert os.path.exists(tarball)
diff --git a/agent/lib/pbench/tests/test_move_results.py b/agent/lib/pbench/tests/test_move_results.py
new file mode 100644
index 0000000000..831fcb9a22
--- /dev/null
+++ b/agent/lib/pbench/tests/test_move_results.py
@@ -0,0 +1,32 @@
+import datetime
+import responses
+
+from agent.lib.pbench.config import AgentConfig
+from agent.lib.pbench.move_results import move_results
+from agent.lib.pbench.tests.common import MockDatetime, mock_agent_config
+
+
+class TestMoveResults:
+    @staticmethod
+    @responses.activate
+    def test_move_results(monkeypatch):
+        monkeypatch.setenv("pbench_tmp", "/tmp")
+        monkeypatch.setenv("full_hostname", "localhost")
+        monkeypatch.setattr(datetime, "datetime", MockDatetime)
+        monkeypatch.setattr(AgentConfig, "get_agent", mock_agent_config)
+
+        responses.add(
+            responses.GET,
+            "http://pbench.example.com/api/v1/host_info",
+            status=200,
+            body="pbench@pbench-server:/srv/pbench/pbench-move-results-receive/fs-version-002",
+        )
+        responses.add(
+            responses.POST, "http://pbench.example.com/api/v1/upload", status=200
+        )
+
+        try:
+            move_results("pbench", "", True)
+        except SystemExit:
+            assert False
+        assert True
diff --git a/doc/CONTRIBUTING.md b/doc/CONTRIBUTING.md
index 618d28a51a..1806697624 100644
--- a/doc/CONTRIBUTING.md
+++ b/doc/CONTRIBUTING.md
@@ -22,6 +22,8 @@ Once tox is installed you can run the unittests:
 
 tox -e agent (for pbench agent)
 tox -e server (for pbench server)
+tox -e pytest_agent (for pytest pbench agent)
+tox -e pytest_server (for pytest pbench server)
 
 # Python formatting
 
diff --git a/pbench/common/configtools.py b/pbench/common/configtools.py
index 63c0fe3d44..835e5d0302 100644
--- a/pbench/common/configtools.py
+++ b/pbench/common/configtools.py
@@ -43,7 +43,6 @@ def file_list(root):
     # insert the directory of the root file at the beginning
     dirlist.insert(0, rootdir)
 
-    # import pdb; pdb.set_trace()
     for d in dirlist:
         for f in files:
             fnm = "%s/%s" % (d, f)
@@ -64,17 +63,19 @@ def init(opts, env_config):
     elif env_config in os.environ:
         conf_file = os.environ[env_config]
     else:
-        return (None, [])
+        return None, []
 
     conffiles = file_list(conf_file)
     conffiles.reverse()
     files = conf.read(conffiles)
 
-    return (conf, files)
+    return conf, files
 
 
-def parse_args(options=[], usage=None):
+def parse_args(_options=None, usage=None):
     """parse_args"""
+    if not _options:
+        _options = []
     if usage:
         parser = OptionParser(usage=usage)
     else:
@@ -91,8 +92,8 @@ def parse_args(options=[], usage=None):
         help="commands logged but not executed",
     )
     # specific options
-    for o in options:
-        parser.add_option(o)
+    for option in _options:
+        parser.add_option(option)
 
     return parser.parse_args()
 
@@ -115,7 +116,7 @@ def parse_range(s):
         prefix = s
         rng = suffix = ""
 
-    return (prefix, suffix, rng)
+    return prefix, suffix, rng
 
 
 def expand_range(s):
@@ -218,7 +219,7 @@ def main(conf, args, opts, files):
         if opts.all:
             for sec in args:
                 if conf.has_section(sec):
-                    print("[%s]" % (sec))
+                    print("[%s]" % sec)
                     items = conf.items(sec)
                     items.sort()
                     for (n, v) in items:
diff --git a/pyproject.toml b/pyproject.toml
index 1e1eec12c1..caa840ac33 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,3 +1,10 @@
+[build-system]
+requires = [
+    "setuptools >= 46.1.3.",
+    "wheel >= 0.34.2",
+]
+build-backend="setuptools.build_meta"
+
 [tool.black]
 skip-string-normalization = false
 exclude = '''
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000000..69f197b981
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,3 @@
+[pytest]
+log_cli=False
+log_level=DEBUG
diff --git a/requirements.txt b/requirements.txt
index 5759697373..ed7dd0285e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,7 @@ ansible
 elasticsearch
 sh
 boto3
+requests
+werkzeug
+flask
+flask-restful
diff --git a/run-unittests b/run-unittests
index 8148650d4a..f73f462414 100755
--- a/run-unittests
+++ b/run-unittests
@@ -4,7 +4,7 @@ prog="$(basename "${0}")"
 progdir="$(realpath -e $(dirname "${0}"))"
 
 # List of tox environments to run.
-tox_envs="lint py3 agent server"
+tox_envs="lint py3 pytest_agent agent pytest_server server"
 
 for toxenv in ${tox_envs}; do
     if [[ -d ${progdir}/.tox/${toxenv} ]]; then
diff --git a/server/lib/config/pbench-server-default.cfg b/server/lib/config/pbench-server-default.cfg
index 7744c2f50a..0888bc4df4 100644
--- a/server/lib/config/pbench-server-default.cfg
+++ b/server/lib/config/pbench-server-default.cfg
@@ -61,6 +61,13 @@ pbench-move-results-receive-versions = 002
 pbench-receive-dir-prefix = %(pbench-local-dir)s/pbench-move-results-receive/fs-version
 pbench-quarantine-dir = %(pbench-local-dir)s/quarantine
 
+# pbench-server rest api variables
+rest_port = 8001
+rest_version = 1
+rest_max_content_length = 100 * 1024 * 1024
+rest_uri = /api/v%(rest_version)s
+rest_log = %(pbench-logs-dir)/pbench-server.log
+
 # WARNING - the pbench-server.cfg file should provide a definition of
 # pbench-backup-dir, e.g.:
 #     pbench-backup-dir = %(pbench-local-dir)s/archive.backup
diff --git a/server/lib/config/pbench-server.cfg.example b/server/lib/config/pbench-server.cfg.example
index cf2e128760..a8c4aa5864 100644
--- a/server/lib/config/pbench-server.cfg.example
+++ b/server/lib/config/pbench-server.cfg.example
@@ -11,8 +11,15 @@ install-dir = /opt/pbench-server
 pbench-top-dir = /srv/pbench
 # Backup dir may be remote NFS mounted
 pbench-backup-dir = /mnt/pbench.archive.backup
+pbench-logs-dir = %(pbench-local-dir)s/logs
 # Add role for sync'ing with satellites
 roles = pbench-maintenance, pbench-prep, pbench-results, pbench-backup, pbench-sync-satellites
+# pbench-server rest api variables
+rest_port = 8001
+rest_version = 1
+rest_max_content_length = 100 * 1024 * 1024
+rest_uri = /api/v%(rest_version)s
+rest_log = %(pbench-logs-dir)s/pbench-server.log
 
 [pbench-server-backup]
 endpoint_url = <the URL of the S3 object store to be used>
diff --git a/server/lib/pbench_server/__init__.py b/server/lib/pbench_server/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/server/lib/pbench_server/config.py b/server/lib/pbench_server/config.py
new file mode 100644
index 0000000000..9df29ef66c
--- /dev/null
+++ b/server/lib/pbench_server/config.py
@@ -0,0 +1,31 @@
+import os
+import sys
+
+from pbench import PbenchConfig, BadConfig
+
+
+class ServerConfig:
+    def __init__(self):
+        self.cfg_name = os.environ.get("_PBENCH_SERVER_CONFIG")
+        if not self.cfg_name:
+            print("{}: ERROR: No config file specified; set CONFIG".format(__name__))
+            sys.exit(1)
+        self.pbench_config = None
+        self.server = None
+
+    def get_pbench_config(self):
+        if not self.pbench_config:
+            try:
+                self.pbench_config = PbenchConfig(self.cfg_name)
+            except BadConfig as e:
+                print(
+                    "{}: {} (config file {})".format(__name__, e, self.cfg_name),
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+        return self.pbench_config
+
+    def get_server_config(self):
+        pc = self.get_pbench_config()
+        self.server = pc.conf["pbench-server"]
+        return self.server
diff --git a/server/lib/pbench_server/pbench_server.py b/server/lib/pbench_server/pbench_server.py
new file mode 100644
index 0000000000..fffb57010e
--- /dev/null
+++ b/server/lib/pbench_server/pbench_server.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+import hashlib
+import os
+
+from flask import request, jsonify, Flask
+from flask_restful import Resource, abort, Api
+from werkzeug.utils import secure_filename
+
+from pbench import get_pbench_logger
+from pbench_server.config import ServerConfig
+
+ALLOWED_EXTENSIONS = {"xz"}
+
+app = None
+
+
+def allowed_file(filename):
+    try:
+        fn = filename.rsplit(".", 1)[1].lower()
+    except IndexError:
+        return False
+    allowed = "." in filename and fn in ALLOWED_EXTENSIONS
+    return allowed
+
+
+def register_endpoints(api, app):
+    api.add_resource(Upload, f"{app.config['REST_URI']}/upload")
+    api.add_resource(HostInfo, f"{app.config['REST_URI']}/host_info")
+
+
+def create_app():
+    global app
+
+    config = ServerConfig()
+
+    app = Flask(__name__)
+    api = Api(app)
+
+    app.config_pbench = config.get_pbench_config()
+    app.config_server = config.get_server_config()
+
+    app.config["PORT"] = app.config_server.get("rest_port")
+    app.config["VERSION"] = app.config_server.get("rest_version")
+    app.config["MAX_CONTENT_LENGTH"] = app.config_server.get("rest_max_content_length")
+    app.config["REST_URI"] = app.config_server.get("rest_uri")
+    app.config["LOG"] = app.config_server.get("rest_log")
+
+    app.logger = get_pbench_logger(__name__, app.config_pbench)
+
+    register_endpoints(api, app)
+
+    return app
+
+
+class HostInfo(Resource):
+    def get(self):
+        global app
+        try:
+            response = jsonify(
+                dict(
+                    message=f"{app.config_server.get('user')}@{app.config_server.get('host')}"
+                    f":{app.config_server.get('pbench-receive-dir-prefix')}-002"
+                )
+            )
+        except Exception:
+            app.logger.debug("There was something wrong constructing the host info.")
+            abort(400, message="There was something wrong with your request")
+        response.status_code = 200
+        return response
+
+
+class Upload(Resource):
+    def post(self):
+        global app
+        if not request.headers.get("filename"):
+            app.logger.debug("Missed filename in header")
+            abort(400, message="Missing filename header in request")
+        filename = secure_filename(request.headers.get("filename"))
+
+        if not request.headers.get("md5sum"):
+            app.logger.debug("Missed md5sum in header")
+            abort(400, message="Missing md5sum header in request")
+        md5sum = request.headers.get("md5sum")
+
+        app.logger.debug(f"Receiving file: {filename}")
+        if not allowed_file(filename):
+            app.logger.debug("Bad file extension received")
+            abort(400, message="File extension not supported. Only .xz")
+
+        upload_directory = app.config_server.get("pbench-receive-dir-prefix")
+        if not upload_directory:
+            app.logger.debug("Missing config variable for pbench-receive-dir-prefix")
+            abort(400, message="Missing config variable for pbench-receive-dir-prefix")
+        if not os.path.exists(upload_directory):
+            os.mkdir(upload_directory)
+        full_path = os.path.join(upload_directory, filename)
+
+        try:
+            with open(full_path, "wb") as f:
+                chunk_size = 4096
+                app.logger.debug("Writing chunks")
+                hash_md5 = hashlib.md5()
+
+                while True:
+                    chunk = request.stream.read(chunk_size)
+                    if len(chunk) == 0:
+                        break
+
+                    f.write(chunk)
+                    hash_md5.update(chunk)
+        except Exception:
+            abort(400, message=f"There was something wrong uploading {filename}")
+
+        if hash_md5.hexdigest() != md5sum:
+            abort(400, message=f"md5sum check failed for {filename}")
+
+        response = jsonify(dict(message="File successfully uploaded"))
+        response.status_code = 201
+        return response
+
+
+if __name__ == "__main__":
+    app = create_app()
+    app.run(debug=True, port=app.config["PORT"])
diff --git a/server/lib/pbench_server/tests/common.py b/server/lib/pbench_server/tests/common.py
new file mode 100644
index 0000000000..72c747e59b
--- /dev/null
+++ b/server/lib/pbench_server/tests/common.py
@@ -0,0 +1,7 @@
+temp_dir = "/tmp/"
+filename = "./server/lib/pbench_server/tests/fixtures/upload/log.tar.xz"
+
+
+def mock_get_config_prefix(app, *args, **kwargs):
+    app.config_server["pbench-receive-dir-prefix"] = temp_dir
+    return app.config_server
diff --git a/server/lib/pbench_server/tests/conftest.py b/server/lib/pbench_server/tests/conftest.py
new file mode 100644
index 0000000000..02e25fd13f
--- /dev/null
+++ b/server/lib/pbench_server/tests/conftest.py
@@ -0,0 +1,36 @@
+import os
+import shutil
+import pytest
+
+from pbench_server.config import ServerConfig
+from pbench_server.pbench_server import create_app
+from pbench_server.tests.common import mock_get_config_prefix
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup(request):
+    """Test package setup"""
+    print("Test SETUP")
+    os.makedirs("./srv/pbench/tmp", exist_ok=True)
+    os.makedirs("./srv/pbench/logs", exist_ok=True)
+    os.makedirs("./opt/pbench-server/bin", exist_ok=True)
+    os.makedirs("./opt/pbench-server/lib", exist_ok=True)
+
+    def teardown():
+        """Test package teardown"""
+        print("Test TEARDOWN")
+        shutil.rmtree("./srv/")
+        shutil.rmtree("./opt/")
+
+    request.addfinalizer(teardown)
+
+
+@pytest.fixture
+def client(monkeypatch):
+    app = create_app()
+    monkeypatch.setattr(ServerConfig, "get_server_config", mock_get_config_prefix(app))
+    app.testing = True
+    """A test client for the app."""
+    app_client = app.test_client()
+    app_client.config = app.config
+    return app_client
diff --git a/server/lib/pbench_server/tests/fixtures/config/pbench-server.cfg b/server/lib/pbench_server/tests/fixtures/config/pbench-server.cfg
new file mode 100644
index 0000000000..0ffc481cb3
--- /dev/null
+++ b/server/lib/pbench_server/tests/fixtures/config/pbench-server.cfg
@@ -0,0 +1,212 @@
+# This file describes the elements of deploying the pbench background tools.
+[DEFAULT]
+
+###########################################################################
+# These should be overridden in the env-specific config file.
+default-host = pbench.example.com
+default-user = pbench
+default-group = pbench
+#
+###########################################################################
+
+# WARNING - the pbench-server.cfg file should provide a definition of
+# install-dir, e.g.:
+#     install-dir = /opt/pbench-server
+# We won't define it here by default to avoid unintended behaviors.
+script-dir = %(install-dir)s/bin
+lib-dir = %(install-dir)s/lib
+crontab-dir = %(lib-dir)s/crontab
+lock-dir = %(lib-dir)s/locks
+
+install-dir = ./opt/pbench-server
+
+###########################################################################
+## Deployment section
+###########################################################################
+[pbench-server]
+host=%(default-host)s
+user=%(default-user)s
+group=%(default-group)s
+admin-email=%(user)s@%(host)s
+mailto=%(admin-email)s
+mailfrom=%(user)s@%(host)s
+commit-id=unknown
+
+# See pbench-server-setup documentation for filesystem setup.
+
+# WARNING - the pbench-server.cfg file should provide a definition of
+# pbench-top-dir, e.g.:
+#     pbench-top-dir = /srv/pbench
+# We won't define it here by default to avoid unintended behaviors.
+
+pbench-top-dir = ./srv/pbench
+pbench-archive-version = 001
+pbench-archive-dir = %(pbench-top-dir)s/archive/fs-version-%(pbench-archive-version)s
+
+# By default we unpack tar balls directly into the incoming directory tree.
+pbench-incoming-dir = %(pbench-top-dir)s/public_html/incoming
+
+# By default the local directory is the same as the top directory. You
+# might want to consider placing the local directory on a separate FS
+# to avoid the temporary files, log files, quarantine, and reception files
+# from competing with disk bandwidth and space of the archive, incoming and
+# results hierarchies.
+pbench-local-dir = %(pbench-top-dir)s
+
+pbench-logs-dir = %(pbench-local-dir)s/logs
+pbench-tmp-dir = %(pbench-local-dir)s/tmp
+
+# Which client move-results versions we support
+pbench-move-results-receive-versions = 002
+pbench-receive-dir-prefix = %(pbench-local-dir)s/pbench-move-results-receive/fs-version
+pbench-quarantine-dir = %(pbench-local-dir)s/quarantine
+
+# pbench-server rest api variables
+rest_port = 8001
+rest_version = 1
+rest_max_content_length = 100 * 1024 * 1024
+rest_uri = /api/v%(rest_version)s
+rest_log = %(pbench-logs-dir)s/pbench-server.log
+
+# WARNING - the pbench-server.cfg file should provide a definition of
+# pbench-backup-dir, e.g.:
+#     pbench-backup-dir = %(pbench-local-dir)s/archive.backup
+# We won't define it here by default to avoid unintended behaviors.
+
+# Default roles this pbench server takes on, see crontab roles below.
+roles = pbench-prep, pbench-results, pbench-backup
+
+# We now index status into ES, but in some cases we might still want to
+# use mail.
+use-mail-for-status = no
+
+# Optional server environment definition
+#environment = staging
+
+# Optional setting when we need to set host as an alias for the real
+# host name
+#realhost = ip-127.0.0.1
+
+# When a tarball comes in to this server, what processing should it
+# undergo?  Typically we want to first unpack it, and back it up.
+dispatch-states = TO-UNPACK, TO-BACKUP
+
+# Satellite servers typically do at least these dispatch states:
+#dispatch-states = TO-UNPACK, TO-SYNC
+
+# Production servers typically want to also copy sosreports.
+unpacked-states = TO-INDEX, TO-COPY-SOS
+
+# Satellite servers typically only want to unpack, so just define empty.
+#unpacked-states =
+
+# Upper and lower bounds in MB bytes
+[pbench-unpack-tarballs/small]
+upperbound = 130
+[pbench-unpack-tarballs/medium]
+lowerbound = 130
+upperbound = 240
+[pbench-unpack-tarballs/large]
+lowerbound = 240
+upperbound = 820
+[pbench-unpack-tarballs/huge]
+lowerbound = 820
+
+# We need to install some stuff in the apache document root so we
+# either get it directly or look in the config file.
+#
+# N.B. Different distros use different config files.  The following
+# works on Fedora, RHEL, CentOS.
+[apache]
+documentroot = /var/www/html
+configfile = /etc/httpd/conf/httpd.conf
+
+# Defaults for pbench-copy-sosreports
+[sosreports]
+user = vos
+host = pbench-sosreports.example.com
+dir = /path/to/sosreport/dir
+
+# This *has* to agree with the setting in the pbench-agent config file.
+[results]
+webserver = %(default-host)s
+host-info-path-prefix = pbench-results-host-info.versioned/pbench-results-host-info.URL
+host_info_url = http://%(webserver)s/%(host-info-path-prefix)s
+
+[logging]
+logger_type = devlog
+# hostport logger type uses UDP-based logging.
+# logger_host = localhost
+# logger_port = 514
+
+###########################################################################
+# crontab roles
+[pbench-prep]
+host = %(default-host)s
+user = %(default-user)s
+mailfrom = %(user)s@%(host)s
+tasks = pbench-prep-task
+
+[prep-shim-002]
+shim-version = 002
+
+[pbench-results]
+host = %(default-host)s
+user = %(default-user)s
+mailfrom = %(user)s@%(host)s
+tasks = pbench-dispatch, pbench-unpack-tarballs-small, pbench-unpack-tarballs-medium, pbench-unpack-tarballs-large, pbench-unpack-tarballs-huge, pbench-copy-sosreports, pbench-edit-prefixes, pbench-index, pbench-index-tool-data
+
+[pbench-backup]
+host = %(default-host)s
+user = %(default-user)s
+mailfrom = %(user)s@%(host)s
+tasks = pbench-backup-tarballs, pbench-verify-backup-tarballs
+
+###########################################################################
+# crontab tasks
+[pbench-prep-task]
+# This is a template that is expanded by pbench-server-activate-create-crontab
+crontab =  * * * * *  flock -n %(lock-dir)s/pbench-prep-$SHIM_VERSION.lock %(script-dir)s/pbench-server-prep-shim-$SHIM_VERSION
+
+[pbench-dispatch]
+crontab =  * * * * *  flock -n %(lock-dir)s/pbench-dispatch.lock %(script-dir)s/pbench-dispatch
+
+[pbench-backup-tarballs]
+crontab = 41 4 * * *  flock -n %(lock-dir)s/pbench-backup-tarballs.lock %(script-dir)s/pbench-backup-tarballs
+
+[pbench-verify-backup-tarballs]
+crontab = 59 4 * * *  flock -n %(lock-dir)s/pbench-verify-backup-tarballs.lock %(script-dir)s/pbench-verify-backup-tarballs
+
+[pbench-unpack-tarballs]
+crontab =  * * * * *  flock -n %(lock-dir)s/pbench-unpack-tarballs.lock %(script-dir)s/pbench-unpack-tarballs
+
+[pbench-unpack-tarballs-small]
+crontab =  * * * * *  flock -n %(lock-dir)s/pbench-unpack-tarballs-small.lock %(script-dir)s/pbench-unpack-tarballs small
+
+[pbench-unpack-tarballs-medium]
+crontab =  * * * * *  flock -n %(lock-dir)s/pbench-unpack-tarballs-medium.lock %(script-dir)s/pbench-unpack-tarballs medium
+
+[pbench-unpack-tarballs-large]
+crontab =  * * * * *  flock -n %(lock-dir)s/pbench-unpack-tarballs-large.lock %(script-dir)s/pbench-unpack-tarballs large
+
+[pbench-unpack-tarballs-huge]
+crontab =  * * * * *  flock -n %(lock-dir)s/pbench-unpack-tarballs-huge.lock %(script-dir)s/pbench-unpack-tarballs huge
+
+[pbench-copy-sosreports]
+crontab = 41 * * * *  flock -n %(lock-dir)s/pbench-copy-sosreports.lock %(script-dir)s/pbench-copy-sosreports
+
+[pbench-edit-prefixes]
+crontab =  * * * * *  flock -n %(lock-dir)s/pbench-edit-prefixes.lock %(script-dir)s/pbench-edit-prefixes
+
+[pbench-index]
+crontab =  * * * * *  flock -n %(lock-dir)s/pbench-index.lock %(script-dir)s/pbench-index
+
+[pbench-index-tool-data]
+crontab =  * * * * *  flock -n %(lock-dir)s/pbench-index-tool-data.lock %(script-dir)s/pbench-index --tool-data
+
+[pbench-sync]
+# This is a template that is expanded by pbench-server-activate-create-crontab
+crontab =  * * * * *  flock -n %(lock-dir)s/$SATELLITE_LOCK %(script-dir)s/pbench-sync-satellite $SATELLITE_CONFIG
+
+[pbench-sync-cleanup]
+crontab = 37 * * * *  flock -n %(lock-dir)s/pbench-satellite-cleanup.lock %(script-dir)s/pbench-satellite-cleanup
diff --git a/server/lib/pbench_server/tests/fixtures/upload/log.tar.xz b/server/lib/pbench_server/tests/fixtures/upload/log.tar.xz
new file mode 100644
index 0000000000000000000000000000000000000000..e3c9e1f6bc8479122090a98322f5269c609d9162
GIT binary patch
literal 640
zcmV-`0)PD<iwFRz?@wL;1MQW|j?*v{hP(DtcsC&7)R){vS+axG?5e7A6Q_-u#KCqt
zz|&)=FhfU01GAVB()qH8buK=3{2#|Eb5dsMly!F5v|rt1lwJ1nsJ4DEb#Ci+zwA+`
z_lsvhJ!u+;H_H<G`+J6^Y{5jz1y2%2y`A$oNfwwsr`W$TStkv~i@lM})Y(7AJ>~yT
z`KWf7rI?B`p(W+G)0ONF+B*DMxjSu+pX4C-%B&g7@=b}~w?m^pV7z8Y&hl7L%A=GQ
z#Rk`hYK-xlyKdx`zAZ7vgY1=^^@VR3jl#ojUVFp91@>?HzgJGy(#dCD08jKk6GEK(
zKT9)_iO~N^2L1mctjC=is>23_X2mOfdGg3srX8KPpTo%H^+M)hFU=8EdT83ccMtWE
z1Pu{}yimeKoU)wIgm7`|FBi^A!I2qz;?-tbJ@^P3C6uszRuoE7o}kt#Bc1nI?_}>V
zL;hsQPIeusjG8^fwf;(RvLELPoD#|jXScK{6H&%llqRX51ww15b!Rsy++<-o*^LKz
zYP(LMz0p%huT>-cE`P-?A41M);e(JzxvjU$@Hq%+2fy5t>{bHx<2CE3Of%bzs<f$z
z9-UpC$CyZOZ;bk~^8Y(Dw)K?>yxr*PY%tR}Z{xB+aG&~XwXK)*Ba?6J-PP5jwV8%Z
zI7(-<-=w+NauiP7ezTEL^+CF*(me_t<F-EQ`Y}f$&r+U?j3PDE*W-98iYO8O2a?5%
zqEO^sWXQ<7t!=*t73Fd47x<uzCEGmcWcyRIY|h`_Yt?p2jR?JVYXJZN0000000000
a00000000000002Mi}3^c!)qS^C;$Myd_{2p

literal 0
HcmV?d00001

diff --git a/server/lib/pbench_server/tests/fixtures/upload/log.tar.xz.md5.check b/server/lib/pbench_server/tests/fixtures/upload/log.tar.xz.md5.check
new file mode 100644
index 0000000000..e781305e5a
--- /dev/null
+++ b/server/lib/pbench_server/tests/fixtures/upload/log.tar.xz.md5.check
@@ -0,0 +1 @@
+d41d8cd98f00b204e9800998ecf8427e
\ No newline at end of file
diff --git a/server/lib/pbench_server/tests/test_requests.py b/server/lib/pbench_server/tests/test_requests.py
new file mode 100644
index 0000000000..bdeb7e2e29
--- /dev/null
+++ b/server/lib/pbench_server/tests/test_requests.py
@@ -0,0 +1,55 @@
+import os
+
+import pytest
+from werkzeug.utils import secure_filename
+
+from pbench_server.tests.common import (
+    filename,
+    temp_dir,
+)
+
+
+class TestHostInfo:
+    @staticmethod
+    def test_host_info(client):
+        response = client.get(f"{client.config['REST_URI']}/host_info")
+        assert response.status_code == 200
+        assert response.json.get("message") == "pbench@pbench.example.com" ":/tmp/-002"
+
+
+class TestUpload:
+    @staticmethod
+    def test_missing_filename_header_upload(client):
+        response = client.post(f"{client.config['REST_URI']}/upload")
+        assert response.status_code == 400
+        assert response.json.get("message") == "Missing filename header in request"
+
+    @staticmethod
+    def test_missing_md5sum_header_upload(client):
+        response = client.post(
+            f"{client.config['REST_URI']}/upload", headers={"filename": filename}
+        )
+        assert response.status_code == 400
+        assert response.json.get("message") == "Missing md5sum header in request"
+
+    @staticmethod
+    @pytest.mark.parametrize("bad_extension", ("test.tar.bad", "test.tar", "test.tar."))
+    def test_bad_extension_upload(client, bad_extension):
+        response = client.post(
+            f"{client.config['REST_URI']}/upload",
+            headers={"filename": bad_extension, "md5sum": "md5sum"},
+        )
+        assert response.status_code == 400
+        assert response.json.get("message") == "File extension not supported. Only .xz"
+
+    @staticmethod
+    def test_upload(client):
+        with open(f"{filename}.md5.check") as md5sum_check:
+            md5sum = md5sum_check.read()
+
+        response = client.post(
+            f"{client.config['REST_URI']}/upload",
+            headers={"filename": filename, "md5sum": md5sum},
+        )
+        assert response.status_code == 201
+        assert os.path.exists(os.path.join(temp_dir, secure_filename(filename)))
diff --git a/server/lib/systemd/pbench-server.service b/server/lib/systemd/pbench-server.service
new file mode 100644
index 0000000000..245e7d3330
--- /dev/null
+++ b/server/lib/systemd/pbench-server.service
@@ -0,0 +1,25 @@
+# copy to: /etc/systemd/system/pbench-server.service
+# and run: systemctl reload-daemon
+# enable: systemctl enable pbench-server
+# start: systemctl start pbench-server
+
+[Unit]
+Description = PBench Daemon
+Documentation = https://github.com/distributed-system-analysis/pbench
+After=network.target httpd.service
+
+[Service]
+Type = simple
+ExecStart = /opt/pbench-server/utils/pbench_server.py
+ExecStop = /usr/bin/pkill pbench-server
+Restart = always
+StartLimitInterval = 60
+StartLimitBurst = 10
+# this is required for newer libraries
+# set appropriately for your environment
+# or use systemctl edit pbench-server to put overrides
+# in another location
+Environment="PYTHONPATH=$PYTHONPATH:/opt/pbench-server/"
+
+[Install]
+WantedBy = multi-user.target
diff --git a/test-requirements.txt b/test-requirements.txt
index 4645a94eba..88dd6ee96a 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,5 +1,6 @@
 flake8
 pre-commit
+responses
 mock
 pytest-mock>=1.10.4, < 2
 pytest-sugar>=0.9.2, < 1
diff --git a/tox.ini b/tox.ini
index 56cde83a40..48978b2da5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,18 +3,18 @@ minversion = 3.5.0
 skipdist = True
 skip_missing_interpreters = True
 isolated_build = True
-envlist = lint,server,agent,py3
+envlist = lint,server,agent,pytest_server,pytest_agent,py3
 
 [testenv]
 usedevelop = True
 install_command = pip install -U {opts} {packages}
 setenv =
-   VIRTUAL_ENV={envdir}
+    VIRTUAL_ENV = {envdir}
 deps =
    -r{toxinidir}/requirements.txt
    -r{toxinidir}/test-requirements.txt
 envars =
-   PATH = {env:PATH}:{toxinidir}/bin
+    PATH = {env:PATH}:{toxinidir}/bin
 whitelist_externals =
     bash
     rm
@@ -43,6 +43,24 @@ commands_pre =
 commands =
   bash -c "./agent/run-unittests"
 
+[testenv:pytest_server]
+setenv =
+  PYTHONPATH = {envdir}{:}{toxinidir}{:}{toxinidir}/server/lib/
+  _PBENCH_SERVER_CONFIG={toxinidir}/server/lib/pbench_server/tests/fixtures/config/pbench-server.cfg
+commands_pre =
+  find {toxinidir} -type f -not -path '{toxinidir}/.tox/*' -path '*/__pycache__/*' -name '*.py[c|o]' -delete
+commands =
+  pytest -s ./server/lib/pbench_server/tests
+
+[testenv:pytest_agent]
+setenv =
+  PYTHONPATH = {envdir}{:}{toxinidir}{:}{toxinidir}/agent/lib/
+  _PBENCH_AGENT_CONFIG={toxinidir}/agent/lib/pbench/tests/fixtures/config/pbench-agent.cfg
+commands_pre =
+  find {toxinidir} -type f -not -path '{toxinidir}/.tox/*' -path '*/__pycache__/*' -name '*.py[c|o]' -delete
+commands =
+  pytest ./agent/lib/pbench/tests
+
 [testenv:lint]
 description = Runs all linting tasks
 commands =