Add support for reading module metadata

Signed-off-by: Stephen Gallagher <[email protected]>

CMakeLists.txt

@@ -120,6 +120,14 @@ IF (WITH_ZCHUNK)
     SET (CMAKE_C_FLAGS_DEBUG    "${CMAKE_C_FLAGS_DEBUG} -DWITH_ZCHUNK")
 ENDIF (WITH_ZCHUNK)
 
+OPTION (WITH_LIBMODULEMD "Build with libmodulemd support" ON)
+IF (WITH_LIBMODULEMD)
+	find_package(LIBMODULEMD REQUIRED)
+	include_directories(${LIBMODULEMD_INCLUDE_DIRS})
+	SET (CMAKE_C_FLAGS          "${CMAKE_C_FLAGS} -DWITH_LIBMODULEMD")
+    SET (CMAKE_C_FLAGS_DEBUG    "${CMAKE_C_FLAGS_DEBUG} -DWITH_LIBMODULEMD")
+ENDIF (WITH_LIBMODULEMD)
+
 # Threaded XZ Compression
 # Note: This option is disabled by default, because Createrepo_c
 # parallelize a lot of tasks (including compression) by default, this

README.md

@@ -16,6 +16,7 @@ Package build requires - Pkg name in Fedora/Ubuntu:
 * file (http://www.darwinsys.com/file/) - file-devel/libmagic-dev
 * glib2 (http://developer.gnome.org/glib/) - glib2-devel/libglib2.0-dev
 * libcurl (http://curl.haxx.se/libcurl/) - libcurl-devel/libcurl4-openssl-dev
+* libmodulemd (https://github.com/fedora-modularity/libmodulemd/) - libmodulemd-devel/
 * libxml2 (http://xmlsoft.org/) - libxml2-devel/libxml2-dev
 * python (http://python.org/) - python3-devel/libpython3-dev
 * rpm (http://www.rpm.org/) - rpm-devel/librpm-dev
@@ -102,6 +103,13 @@ cause degradation of performance.
 
 Build with zchunk support (Default: ON)
 
+### ``-DWITH_LIBMODULEMD=ON``
+
+Build with libmodulemd support (Default: ON)
+
+Adds support for working with repos containing
+[Fedora Modularity](https://docs.fedoraproject.org/en-US/modularity/) metadata.
+
 
 ## Build tarball
 

cmake/Modules/FindLIBMODULEMD.cmake

@@ -0,0 +1,40 @@
+# Try to find libmodulemd 2.3+
+#
+#  LIBMODULEMD_FOUND - system has libmodulemd
+#  LIBMODULEMD_INCLUDE_DIRS - the libmodulemd include directory
+#  LIBMODULEMD_LIBRARIES - Link these to use libmodulemd
+#
+#  Copyright (c) 2019 Stephen Gallagher <[email protected]>
+
+#  Redistribution and use is allowed according to the terms of the New
+#  BSD license.
+
+IF ( LIBMODULEMD_LIBRARIES AND LIBMODULEMD_INCLUDE_DIRS )
+  # in cache already
+  SET(LIBMODULEMD_FOUND TRUE)
+ELSE ( LIBMODULEMD_LIBRARIES AND LIBMODULEMD_INCLUDE_DIRS )
+
+	INCLUDE(FindPkgConfig)
+
+	IF ( LIBMODULEMD_FIND_REQUIRED )
+		SET ( _pkgconfig_REQUIRED "REQUIRED" )
+	ELSE ( LIBMODULEMD_FIND_REQUIRED )
+		SET ( _pkgconfig_REQUIRED "" )
+	endif ( LIBMODULEMD_FIND_REQUIRED )
+
+	PKG_SEARCH_MODULE ( LIBMODULEMD ${_pkgconfig_REQUIRED} modulemd-2.0>=2.3 )
+
+	IF (LIBMODULEMD_FOUND)
+    IF (NOT LIBMODULEMD_FIND_QUIETLY)
+      MESSAGE (STATUS "Found libmodulemd: ${LIBMODULEMD_LIBRARIES} ${LIBMODULEMD_INCLUDE_DIRS}")
+    ENDIF (NOT LIBMODULEMD_FIND_QUIETLY)
+  ELSE (LIBMODULEMD_FOUND)
+    IF (LIBMODULEMD_FIND_REQUIRED)
+      MESSAGE (SEND_ERROR "Could not find libmodulemd")
+    ENDIF (LIBMODULEMD_FIND_REQUIRED)
+  ENDIF (LIBMODULEMD_FOUND)
+
+  # show the LIBMODULEMD_INCLUDE_DIRS and LIBMODULEMD_LIBRARIES variables only in the advanced view
+  MARK_AS_ADVANCED(LIBMODULEMD_INCLUDE_DIRS LIBMODULEMD_LIBRARIES)
+
+ENDIF ( LIBMODULEMD_LIBRARIES AND LIBMODULEMD_INCLUDE_DIRS )

src/CMakeLists.txt

@@ -67,6 +67,7 @@ TARGET_LINK_LIBRARIES(libcreaterepo_c ${CURL_LIBRARY})
 TARGET_LINK_LIBRARIES(libcreaterepo_c ${EXPAT_LIBRARIES})
 TARGET_LINK_LIBRARIES(libcreaterepo_c ${GLIB2_LIBRARIES})
 TARGET_LINK_LIBRARIES(libcreaterepo_c ${Libmagic_LIBRARY})
+TARGET_LINK_LIBRARIES(libcreaterepo_c ${LIBMODULEMD_LIBRARIES})
 TARGET_LINK_LIBRARIES(libcreaterepo_c ${LIBXML2_LIBRARIES})
 TARGET_LINK_LIBRARIES(libcreaterepo_c ${LZMA_LIBRARIES})
 TARGET_LINK_LIBRARIES(libcreaterepo_c ${OPENSSL_LIBRARIES})

src/error.h

@@ -96,6 +96,8 @@ typedef enum {
         (33) Cannot change blocked signals */
     CRE_ZCK, /*!<
         (34) ZCK library related error */
+    CRE_MODULEMD, /*!<
+        (35) modulemd related error */
     CRE_SENTINEL, /*!<
         (XX) Sentinel */
 } cr_Error;

src/load_metadata.c

@@ -23,6 +23,11 @@
 #include <fcntl.h>
 #include <string.h>
 #include <assert.h>
+
+#ifdef WITH_LIBMODULEMD
+#include <modulemd.h>
+#endif /* WITH_LIBMODULEMD */
+
 #include "error.h"
 #include "package.h"
 #include "misc.h"
@@ -42,6 +47,10 @@ struct _cr_Metadata {
     GHashTable *pkglist_ht; /*!< list of allowed package basenames to load */
     cr_HashTableKeyDupAction dupaction; /*!<
         How to behave in case of duplicated items */
+
+#ifdef WITH_LIBMODULEMD
+    ModulemdModuleIndex *moduleindex; /*!< Module metadata */
+#endif /* WITH_LIBMODULEMD */
 };
 
 cr_HashTableKey
@@ -58,6 +67,15 @@ cr_metadata_hashtable(cr_Metadata *md)
     return md->ht;
 }
 
+#ifdef WITH_LIBMODULEMD
+ModulemdModuleIndex *
+cr_metadata_modulemd(cr_Metadata *md)
+{
+    assert(md);
+    return md->moduleindex;
+}
+#endif /* WITH_LIBMODULEMD */
+
 void
 cr_free_values(gpointer data)
 {
@@ -543,9 +561,93 @@ cr_metadata_load_xml(cr_Metadata *md,
     g_hash_table_destroy(ignored_keys);
     cr_destroy_metadata_hashtable(intern_hashtable);
 
-    return CRE_OK;
+    result = CRE_OK;
+
+#ifdef WITH_LIBMODULEMD
+    if (ml->modulemd_href) {
+      result = cr_metadata_load_modulemd(md, ml, err);
+    }
+#endif /* WITH_LIBMODULEMD */
+
+    return result;
 }
 
+
+static gint
+module_read_fn (void *data,
+                unsigned char *buffer,
+                size_t size,
+                size_t *size_read)
+{
+    int ret;
+    GError *tmp_err = NULL;
+    CR_FILE *cr_file = (CR_FILE *)data;
+
+    ret = cr_read (cr_file, buffer, size, &tmp_err);
+    if (ret == CR_CW_ERR) {
+        g_clear_pointer (&tmp_err, g_error_free);
+        return 0;
+    }
+
+    *size_read = ret;
+    return 1;
+}
+
+#ifdef WITH_LIBMODULEMD
+int
+cr_metadata_load_modulemd(cr_Metadata *md,
+                          struct cr_MetadataLocation *ml,
+                          GError **err)
+{
+    int ret;
+    gboolean result;
+    GError *tmp_err = NULL;
+    CR_FILE *modulemd = NULL;
+    g_autoptr (GPtrArray) failures = NULL;
+
+    md->moduleindex = modulemd_module_index_new();
+    if (!md->moduleindex) {
+        g_set_error (err, ERR_DOMAIN, CRE_MEMORY,
+                     "Could not allocate module index");
+        return CRE_MEMORY;
+    }
+
+    /* Open the metadata location */
+    modulemd = cr_open(ml->modulemd_href,
+                       CR_CW_MODE_READ,
+                       CR_CW_AUTO_DETECT_COMPRESSION,
+                       &tmp_err);
+    if (tmp_err) {
+        int code = tmp_err->code;
+        g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ",
+                                   ml->modulemd_href);
+        return code;
+    }
+
+    result = modulemd_module_index_update_from_custom (md->moduleindex,
+                                                       module_read_fn,
+                                                       modulemd,
+                                                       TRUE,
+                                                       &failures,
+                                                       &tmp_err);
+    if (!result) {
+        g_propagate_error (err, tmp_err);
+        return CRE_MODULEMD;
+    }
+
+    ret = CRE_OK;
+
+    cr_close(modulemd, &tmp_err);
+    if (tmp_err) {
+        ret = tmp_err->code;
+        g_propagate_prefixed_error(err, tmp_err, "Error while closing: ");
+
+    }
+
+    return ret;
+}
+#endif /* WITH_LIBMODULEMD */
+
 int
 cr_metadata_locate_and_load_xml(cr_Metadata *md,
                                 const char *repopath,
@@ -566,6 +668,10 @@ cr_metadata_locate_and_load_xml(cr_Metadata *md,
     }
 
     ret = cr_metadata_load_xml(md, ml, err);
+    if (ret != CRE_OK) {
+        return ret;
+    }
+
     cr_metadatalocation_free(ml);
 
     return ret;

src/locate_metadata.c

@@ -116,6 +116,8 @@ cr_parse_repomd(const char *repomd_path,
             mdloc->cgroupfile_href = full_location_href;
         else if (!g_strcmp0(record->type, "updateinfo"))
             mdloc->updateinfo_href = full_location_href;
+        else if (!g_strcmp0(record->type, "modules"))
+            mdloc->modulemd_href = full_location_href;
         else
             g_free(full_location_href);
     }
@@ -305,5 +307,16 @@ cr_locate_metadata(const char *repopath, gboolean ignore_sqlite, GError **err)
         return NULL;
     }
 
+#ifndef WITH_LIBMODULEMD
+    if (ret->modulemd_href) {
+        g_set_error (err,
+                     ERR_DOMAIN,
+                     CRE_MODULEMD,
+                     "Module metadata found in repository, but createrepo_c "
+                     "was not compiled with libmodulemd support.");
+        g_clear_pointer (&ret, cr_metadatalocation_free);
+    }
+#endif /* ! WITH_LIBMODULEMD */
+
     return ret;
 }

src/locate_metadata.h

@@ -43,6 +43,7 @@ struct cr_MetadataLocation {
     char *groupfile_href;       /*!< path to groupfile */
     char *cgroupfile_href;      /*!< path to compressed groupfile */
     char *updateinfo_href;      /*!< path to updateinfo */
+    char *modulemd_href;        /*!< path to modulemd */
     char *repomd;               /*!< path to repomd.xml */
     char *original_url;         /*!< original path of repo from commandline
                                      param */

src/metadata_internal.h

@@ -0,0 +1,43 @@
+/* createrepo_c - Library of routines for manipulation with repodata
+ * Copyright (C) 2013  Tomas Mlcoch
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+ * USA.
+ */
+
+#ifndef __C_CREATEREPOLIB_METADATA_INTERNAL_H__
+#define __C_CREATEREPOLIB_METADATA_INTERNAL_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef WITH_LIBMODULEMD
+#include <modulemd.h>
+#include "load_metadata.h"
+
+/** Return module metadata from a cr_Metadata
+ * @param md        cr_Metadata object.
+ * @return          Pointer to internal ModulemdModuleIndex.
+ */
+ModulemdModuleIndex *cr_metadata_modulemd(cr_Metadata *md);
+
+#endif /* WITH_LIBMODULEMD */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __C_CREATEREPOLIB_METADATA_INTERNAL_H__ */

tests/fixtures.h

@@ -30,6 +30,7 @@
 #define TEST_REPO_00                    TEST_DATA_PATH"repo_00/"
 #define TEST_REPO_01                    TEST_DATA_PATH"repo_01/"
 #define TEST_REPO_02                    TEST_DATA_PATH"repo_02/"
+#define TEST_REPO_03                    TEST_DATA_PATH"repo_03/"
 #define TEST_FILES_PATH                 TEST_DATA_PATH"test_files/"
 #define TEST_UPDATEINFO_FILES_PATH      TEST_DATA_PATH"updateinfo_files/"
 
@@ -50,6 +51,13 @@
 #define TEST_REPO_02_FILELISTS  TEST_REPO_02"repodata/3b7e6ecd01af9cb674aff6458186911d7081bb5676d5562a21a963afc8a8bcc7-filelists.xml.gz"
 #define TEST_REPO_02_OTHER      TEST_REPO_02"repodata/ab5d3edeea50f9b4ec5ee13e4d25c147e318e3a433dbabc94d3461f58ac28255-other.xml.gz"
 
+// REPO_03 is a copy of REPO_01 with some module metadata
+#define TEST_REPO_03_REPOMD     TEST_REPO_03"repodata/repomd.xml"
+#define TEST_REPO_03_PRIMARY    TEST_REPO_03"repodata/1cb61ea996355add02b1426ed4c1780ea75ce0c04c5d1107c025c3fbd7d8bcae-primary.xml.gz"
+#define TEST_REPO_03_FILELISTS  TEST_REPO_03"repodata/95a4415d859d7120efb6b3cf964c07bebbff9a5275ca673e6e74a97bcbfb2a5f-filelists.xml.gz"
+#define TEST_REPO_03_OTHER      TEST_REPO_03"repodata/ef3e20691954c3d1318ec3071a982da339f4ed76967ded668b795c9e070aaab6-other.xml.gz"
+#define TEST_REPO_03_MODULEMD   TEST_REPO_03"repodata/a850093e240506c728d6ce26a6fc51d6a7fe10730c67988d13afa7dd82df82d5-modules.yaml.xz"
+
 // Modified repo files (MFR)
 
 #define TEST_MRF_BAD_TYPE_FIL   TEST_MODIFIED_REPO_FILES_PATH"bad_file_type-filelists.xml"