Merge pull request #59 from mrc-ide/mrc-5041

Add endpoint to run git fetch on outpack root

.github/workflows/python.yaml

@@ -60,13 +60,13 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - {os: ubuntu-latest,  target: 'x86_64'}
-          - {os: ubuntu-latest,  target: 'x86'}
-          - {os: ubuntu-latest,  target: 'aarch64'}
-          - {os: windows-latest, target: 'x64'}
-          - {os: windows-latest, target: 'x86'}
-          - {os: macos-latest,   target: 'x86_64'}
-          - {os: macos-latest,   target: 'aarch64'}
+          - { os: ubuntu-latest,  target: 'x86_64' }
+          - { os: ubuntu-latest,  target: 'x86' }
+          - { os: ubuntu-latest,  target: 'aarch64' }
+          - { os: windows-latest, target: 'x64' }
+          - { os: windows-latest, target: 'x86' }
+          - { os: macos-latest,   target: 'x86_64' }
+          - { os: macos-latest,   target: 'aarch64' }
 
     runs-on: ${{ matrix.config.os }}
     name: Build wheels for ${{ matrix.config.os }} (${{ matrix.config.target }})
@@ -82,8 +82,19 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.config.target }}
-          args: --release --out dist
+          args: --release --out dist --features openssl-vendored
           manylinux: manylinux2014
+          before-script-linux: |
+            # If we're running on RHEL centos, install needed packages.
+            if command -v yum &> /dev/null; then
+              yum update -y && yum install -y perl-core libatomic
+            
+              # If we're running on i686 we need to symlink libatomic
+              # in order to build openssl with -latomic flag.
+              if [[ ! -d "/usr/lib64" ]]; then
+                ln -s /usr/lib/libatomic.so.1 /usr/lib/libatomic.so
+              fi
+            fi
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -107,10 +118,10 @@ jobs:
       # This permission is needed for the workflow to authenticate against PyPI
       id-token: write
     steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v3
-      with:
-        name: python-artifacts
-        path: dist/
-    - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Download all the dists
+        uses: actions/download-artifact@v3
+        with:
+          name: python-artifacts
+          path: dist/
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

Cargo.toml

@@ -35,6 +35,7 @@ tokio-util = { version = "0.7.10", features = ["io"] }
 futures = "0.3.30"
 tower = "0.4.13"
 mime = "0.3.17"
+git2 = { version = "0.18.2" }
 
 [dev-dependencies]
 assert_cmd = "2.0.6"
@@ -46,6 +47,8 @@ tar = "0.4.38"
 chrono = "0.4.33"
 rand = "0.8.5"
 tracing-capture = "0.1.0"
+test-utils = { path = "test-utils" }
 
 [features]
-python = [ "dep:pyo3" ]
+python = ["dep:pyo3"]
+openssl-vendored = ["git2/vendored-openssl"]

src/api.rs

@@ -1,27 +1,27 @@
+use std::any::Any;
+use std::io::ErrorKind;
+use std::path::{Path, PathBuf};
+
 use anyhow::{bail, Context};
 use axum::extract::rejection::JsonRejection;
 use axum::extract::{self, Query, State};
 use axum::response::IntoResponse;
 use axum::response::Response;
 use axum::{Json, Router};
 use serde::{Deserialize, Serialize};
-use std::any::Any;
-use std::io::ErrorKind;
-use std::path::{Path, PathBuf};
 use tower_http::catch_panic::CatchPanicLayer;
 use tower_http::request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer};
 use tower_http::trace::TraceLayer;
 
-use crate::config;
 use crate::hash;
 use crate::location;
 use crate::metadata;
 use crate::metrics;
-use crate::store;
-
 use crate::outpack_file::OutpackFile;
 use crate::responses::{OutpackError, OutpackSuccess};
+use crate::store;
 use crate::upload::{Upload, UploadLayer};
+use crate::{config, git};
 
 type OutpackResult<T> = Result<OutpackSuccess<T>, OutpackError>;
 
@@ -68,6 +68,7 @@ async fn list_location_metadata(
 struct KnownSince {
     known_since: Option<f64>,
 }
+
 async fn get_metadata_since(
     root: State<PathBuf>,
     query: Query<KnownSince>,
@@ -156,6 +157,12 @@ async fn add_packet(
         .map(OutpackSuccess::from)
 }
 
+async fn git_fetch(root: State<PathBuf>) -> Result<OutpackSuccess<()>, OutpackError> {
+    git::git_fetch(&root)
+        .map_err(OutpackError::from)
+        .map(OutpackSuccess::from)
+}
+
 #[derive(Serialize, Deserialize)]
 struct Ids {
     ids: Vec<String>,
@@ -236,6 +243,7 @@ pub fn api(root: &Path) -> anyhow::Result<Router> {
         .route("/packit/metadata", get(get_metadata_since))
         .route("/file/:hash", get(get_file).post(add_file))
         .route("/packet/:hash", post(add_packet))
+        .route("/git/fetch", post(git_fetch))
         .route("/metrics", get(|| async move { metrics::render(registry) }))
         .fallback(not_found)
         .with_state(root.to_owned());

src/git.rs

@@ -0,0 +1,45 @@
+use std::path::Path;
+
+use git2::Repository;
+
+pub fn git_fetch(root: &Path) -> Result<(), git2::Error> {
+    let repo = Repository::open(root)?;
+    let mut remote = repo.find_remote("origin")?;
+    let ref_specs_iter = remote.fetch_refspecs()?;
+    let ref_specs: Vec<&str> = ref_specs_iter.iter().map(|spec| spec.unwrap()).collect();
+    remote.fetch(&ref_specs, None, None)?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use test_utils::{git_get_latest_commit, git_remote_branches, initialise_git_repo};
+
+    use super::*;
+
+    #[test]
+    fn can_perform_git_fetch() {
+        let test_git = initialise_git_repo(None);
+
+        let remote_ref = git_get_latest_commit(&test_git.remote, "HEAD");
+        let initial_ref = git_get_latest_commit(&test_git.local, "refs/remotes/origin/HEAD");
+        assert_ne!(
+            initial_ref.message().unwrap(),
+            remote_ref.message().unwrap()
+        );
+
+        let initial_branches = git_remote_branches(&test_git.local);
+        assert_eq!(initial_branches.count(), 2); // HEAD and main
+
+        git_fetch(&test_git.dir.path().join("local")).unwrap();
+
+        let post_fetch_ref = git_get_latest_commit(&test_git.local, "refs/remotes/origin/HEAD");
+        assert_eq!(
+            post_fetch_ref.message().unwrap(),
+            remote_ref.message().unwrap()
+        );
+
+        let post_fetch_branches = git_remote_branches(&test_git.local);
+        assert_eq!(post_fetch_branches.count(), 3); // HEAD, main and other
+    }
+}

src/lib.rs

@@ -7,6 +7,7 @@ pub mod index;
 pub mod init;
 pub mod query;
 
+mod git;
 mod hash;
 mod location;
 mod metadata;

src/query/python.rs

@@ -97,7 +97,7 @@ enum TestOperator {
 }
 
 #[pyfunction]
-fn parse_query<'a>(py: Python, input: &'a str) -> PyResult<PyObject> {
+fn parse_query(py: Python, input: &str) -> PyResult<PyObject> {
     convert_query(py, crate::query::parse_query(input)?)
 }