diff --git a/backend/addcorpus/python_corpora/corpus.py b/backend/addcorpus/python_corpora/corpus.py index 3b0e2594a..220a427bc 100644 --- a/backend/addcorpus/python_corpora/corpus.py +++ b/backend/addcorpus/python_corpora/corpus.py @@ -79,6 +79,23 @@ def category(self): ''' raise NotImplementedError('CorpusDefinition missing category') + ''' + Directory where source data is located + If neither `data_directory` nor `data_url` is set to valid paths, this corpus cannot be indexed + ''' + data_directory = None + + ''' + URL where source data is located + If neither `data_directory` nor `data_url` is set to valid paths, this corpus cannot be indexed + ''' + data_url = None + + ''' + If connecting to the data URL requires and API key, it needs to be set here + ''' + data_api_key = None + @property def es_index(self): ''' diff --git a/backend/corpora/gallica/conftest.py b/backend/corpora/gallica/conftest.py new file mode 100644 index 000000000..e1d7e2fef --- /dev/null +++ b/backend/corpora/gallica/conftest.py @@ -0,0 +1,35 @@ +import os + +import pytest + +here = os.path.abspath(os.path.dirname(__file__)) + + +@pytest.fixture() +def gallica_corpus_settings(settings): + settings.CORPORA = { + "figaro": os.path.join(here, "figaro.py"), + } + + +class MockResponse(object): + def __init__(self, filepath): + self.mock_content_file = filepath + + @property + def content(self): + with open(self.mock_content_file, "r") as f: + return f.read() + + +def mock_response(url: str) -> MockResponse: + if url.endswith("date"): + filename = os.path.join(here, "tests", "data", "figaro", "Years.xml") + elif "&" in url: + filename = os.path.join(here, "tests", "data", "figaro", "Issues.xml") + elif "?" in url: + filename = os.path.join(here, "tests", "data", "figaro", "OAIRecord.xml") + elif url.endswith("texteBrut"): + filename = os.path.join(here, "tests", "data", "figaro", "RoughText.html") + return MockResponse(filename) + diff --git a/backend/corpora/gallica/figaro.py b/backend/corpora/gallica/figaro.py new file mode 100644 index 000000000..32ac4347b --- /dev/null +++ b/backend/corpora/gallica/figaro.py @@ -0,0 +1,55 @@ +from datetime import datetime +from typing import Union + +from django.conf import settings +from ianalyzer_readers.xml_tag import Tag +from ianalyzer_readers.extract import XML + +from addcorpus.python_corpora.corpus import FieldDefinition +from addcorpus.es_mappings import ( + keyword_mapping, +) + +from corpora.gallica.gallica import Gallica + + +def join_issue_strings(issue_description: Union[list[str], None]) -> Union[str, None]: + if issue_description: + return "".join(issue_description[:2]) + + +class Figaro(Gallica): + title = "Le Figaro" + description = "Newspaper archive, 1854-1953" + min_date = datetime(year=1854, month=1, day=1) + max_date = datetime(year=1953, month=12, day=31) + corpus_id = "cb34355551z" + category = "periodical" + es_index = getattr(settings, 'FIGARO_INDEX', 'figaro') + image = "figaro.jpg" + + contributor = FieldDefinition( + name="contributor", + description="Persons who contributed to this issue", + es_mapping=keyword_mapping(enable_full_text_search=True), + extractor=XML(Tag("dc:contributor"), multiple=True), + ) + + issue = FieldDefinition( + name="issue", + description="Issue description", + es_mapping=keyword_mapping(), + extractor=XML( + Tag("dc:description"), multiple=True, transform=join_issue_strings + ), + ) + + def __init__(self): + self.fields = [ + self.content(), + self.contributor, + self.date(self.min_date, self.max_date), + self.identifier(), + self.issue, + self.url(), + ] diff --git a/backend/corpora/gallica/gallica.py b/backend/corpora/gallica/gallica.py new file mode 100644 index 000000000..fa52cce58 --- /dev/null +++ b/backend/corpora/gallica/gallica.py @@ -0,0 +1,162 @@ +from datetime import datetime +import logging +from time import sleep + +from bs4 import BeautifulSoup +from ianalyzer_readers.xml_tag import Tag +from ianalyzer_readers.extract import Metadata, XML +import requests + +from addcorpus.python_corpora.corpus import XMLCorpusDefinition +from addcorpus.python_corpora.corpus import FieldDefinition +from addcorpus.python_corpora.filters import DateFilter +from addcorpus.es_mappings import ( + keyword_mapping, + date_mapping, + main_content_mapping, +) +from addcorpus.es_settings import es_settings + +logger = logging.getLogger('indexing') + +def get_content(content: BeautifulSoup) -> str: + """Return text content in the parsed HTML file from the `texteBrut` request + This is contained in the first

element after the first

element. + """ + text_nodes = content.find("hr").find_next_siblings("p") + return "".join([node.get_text() for node in text_nodes]) + + +def get_publication_id(identifier: str) -> str: + try: + return identifier.split("/")[-1] + except: + return None + + +class Gallica(XMLCorpusDefinition): + + languages = ["fr"] + data_url = "https://gallica.bnf.fr" + corpus_id = "" # each corpus on Gallica has an "ark" id + + @property + def es_settings(self): + return es_settings( + self.languages[:1], stopword_analysis=True, stemming_analysis=True + ) + + def sources(self, start: datetime, end: datetime): + # obtain list of ark numbers + response = requests.get( + f"{self.data_url}/services/Issues?ark=ark:/12148/{self.corpus_id}/date" + ) + year_soup = BeautifulSoup(response.content, "xml") + years = [ + year.string + for year in year_soup.find_all("year") + if int(year.string) >= start.year and int(year.string) <= end.year + ] + for year in years: + try: + response = requests.get( + f"{self.data_url}/services/Issues?ark=ark:/12148/{self.corpus_id}/date&date={year}" + ) + ark_soup = BeautifulSoup(response.content, "xml") + ark_numbers = [ + issue_tag["ark"] for issue_tag in ark_soup.find_all("issue") + ] + sleep(2) + except ConnectionError: + logger.warning(f"Connection error when processing year {year}") + break + + for ark in ark_numbers: + try: + source_response = requests.get( + f"{self.data_url}/services/OAIRecord?ark={ark}" + ) + sleep(2) + except ConnectionError: + logger.warning(f"Connection error encountered in issue {ark}") + break + + if source_response: + try: + content_response = requests.get( + f"{self.data_url}/ark:/12148/{ark}.texteBrut" + ) + sleep(10) + except ConnectionError: + logger.warning( + f"Connection error when fetching full text of issue {ark}" + ) + parsed_content = BeautifulSoup( + content_response.content, "lxml-html" + ) + yield ( + source_response.content, + {"content": parsed_content}, + ) + + def content(self): + return FieldDefinition( + name="content", + description="Content of publication", + display_name="Content", + display_type="text_content", + es_mapping=main_content_mapping( + token_counts=True, + stopword_analysis=True, + stemming_analysis=True, + language=self.languages[0], + ), + extractor=Metadata("content", transform=get_content), + ) + + def date(self, min_date: datetime, max_date: datetime): + return FieldDefinition( + name="date", + display_name="Date", + description="The date of the publication.", + es_mapping=date_mapping(), + extractor=XML( + Tag("dc:date"), + ), + results_overview=True, + search_filter=DateFilter( + min_date, max_date, description="Search only within this time range." + ), + visualizations=["resultscount", "termfrequency"], + csv_core=True, + ) + + def identifier(self): + return FieldDefinition( + name="id", + display_name="Publication ID", + description="Identifier of the publication on Gallica", + es_mapping=keyword_mapping(), + extractor=XML(Tag("dc:identifier"), transform=get_publication_id), + csv_core=True, + ) + + def url(self): + return FieldDefinition( + name="url", + display_name="Source URL", + display_type="url", + description="URL to scan on Gallica", + es_mapping=keyword_mapping(), + extractor=XML(Tag("dc:identifier")), + searchable=False, + ) + + # define fields property so it can be set in __init__ + @property + def fields(self): + return self._fields + + @fields.setter + def fields(self, value): + self._fields = value diff --git a/backend/corpora/gallica/images/figaro.jpg b/backend/corpora/gallica/images/figaro.jpg new file mode 100644 index 000000000..a59554cae Binary files /dev/null and b/backend/corpora/gallica/images/figaro.jpg differ diff --git a/backend/corpora/gallica/tests/__init__.py b/backend/corpora/gallica/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/corpora/gallica/tests/data/figaro/Issues.xml b/backend/corpora/gallica/tests/data/figaro/Issues.xml new file mode 100644 index 000000000..667c8483d --- /dev/null +++ b/backend/corpora/gallica/tests/data/figaro/Issues.xml @@ -0,0 +1,4 @@ + + +01 janvier 1930 + diff --git a/backend/corpora/gallica/tests/data/figaro/OAIRecord.xml b/backend/corpora/gallica/tests/data/figaro/OAIRecord.xml new file mode 100644 index 000000000..2ac45c5a1 --- /dev/null +++ b/backend/corpora/gallica/tests/data/figaro/OAIRecord.xml @@ -0,0 +1,62 @@ + + +all + + +
+oai:bnf.fr:gallica/ark:/12148/bpt6k296099q +2024-06-21 +gallica:corpus:BnPlCo00 +gallica:corpus:Pam1 +gallica:corpus:bresil +gallica:theme:0:07 +gallica:typedoc:periodiques:fascicules +
+ + +https://gallica.bnf.fr/ark:/12148/bpt6k296099q +1930-01-01 +01 janvier 1930 +1930/01/01 (Numéro 1). + +Figaro : journal non politique +Villemessant, Hippolyte de (1810-1879). Directeur de publication +Jouvin, Benoît (1810-1886). Directeur de publication +Figaro (Paris) +texte +text +publication en série imprimée +printed serial +fre +Notice du catalogue : http://catalogue.bnf.fr/ark:/12148/cb34355551z +Bibliothèque nationale de France +domaine public +public domain +http://gallica.bnf.fr/ark:/12148/cb34355551z/date +Appartient à l’ensemble documentaire : BIPFPIG00 +Appartient à l’ensemble documentaire : BIPFPIG63 +Appartient à l’ensemble documentaire : BIPFPIG69 +Appartient à l’ensemble documentaire : Pam1 +Appartient à l’ensemble documentaire : BnPlCo00 +Appartient à l’ensemble documentaire : BnPlCo01 +Appartient à l’ensemble documentaire : FranceBr +Nombre total de vues : 164718 + + + + +bnf.fr +07 +0 +Bibliothèque nationale de France +fascicule +0.0 +Figaro : journal non politique +1930-01-01 +15/10/2007 +false + + + + + diff --git a/backend/corpora/gallica/tests/data/figaro/RoughText.html b/backend/corpora/gallica/tests/data/figaro/RoughText.html new file mode 100644 index 000000000..4501d9320 --- /dev/null +++ b/backend/corpora/gallica/tests/data/figaro/RoughText.html @@ -0,0 +1,2 @@ +Figaro : journal non politique | Gallica

Reminder of your request:


Downloading format: : Text

View 1 to 8 on 8

Number of pages: 8

Full notice

Title : Figaro : journal non politique

Publisher : Figaro (Paris)

Publication date : 1930-01-01

Contributor : Villemessant, Hippolyte de (1810-1879). Directeur de publication

Contributor : Jouvin, Benoît (1810-1886). Directeur de publication

Relationship : http://catalogue.bnf.fr/ark:/12148/cb34355551z

Relationship : https://gallica.bnf.fr/ark:/12148/cb34355551z/date

Type : text

Type : printed serial

Language : french

Format : Nombre total de vues : 164718

Description : 01 janvier 1930

Description : 1930/01/01 (Numéro 1).

Description : Collection numérique : Bibliographie de la presse française politique et d'information générale

Description : Collection numérique : BIPFPIG63

Description : Collection numérique : BIPFPIG69

Description : Collection numérique : Arts de la marionnette

Description : Collection numérique : Commun Patrimoine: bibliothèque numérique du réseau des médiathèques de Plaine commune

Description : Collection numérique : La Commune de Paris

Description : Collection numérique : France-Brésil

Rights : Consultable en ligne

Rights : Public domain

Identifier : ark:/12148/bpt6k296099q

Source : Bibliothèque nationale de France

Provenance : Bibliothèque nationale de France

Online date : 15/10/2007

The text displayed may contain some errors. The text of this document has been generated automatically by an optical character recognition (OCR) program. The +estimated recognition rate for this document is 0%.

SOMMAIRE DE FIGARO PAGE 2. Les Cours, les Ambassades, le Monde et la Ville. Les Echos. La fin du Bulletin vert. 1929-1930.

PAGE 3. La Dernière Heure. Avant la Conférence de La Haye. Les méfaits de la tempête.

PAGE 4. La Vie sportive. Revue de la Presse. Anne Douglas Sedgwick Marthe Ludérac.

PAGE 5. Henri Rebois L'Art espagnol à l'Exposition de Barcelone. Robert Brussel Le Mouvement musical. Guy de Passillé Les Etrennes. Jacques Patin Les Premières. Les Alguazils Courrier des Lettres. Marc Hélys Revues étrangères. PAGE 6. La Bourse La Cote des Valeurs. Le Programme des spectacles.

PAGE 7. Courrier des théâtres. Les Courses LA POLITIQUE

La diplomatie

diff --git a/backend/corpora/gallica/tests/data/figaro/Years.xml b/backend/corpora/gallica/tests/data/figaro/Years.xml new file mode 100644 index 000000000..4ca05725c --- /dev/null +++ b/backend/corpora/gallica/tests/data/figaro/Years.xml @@ -0,0 +1,102 @@ + + +1854 +1855 +1856 +1857 +1858 +1859 +1860 +1861 +1862 +1863 +1864 +1865 +1866 +1867 +1868 +1869 +1870 +1871 +1872 +1873 +1874 +1875 +1876 +1877 +1878 +1879 +1880 +1881 +1882 +1883 +1884 +1885 +1886 +1887 +1888 +1889 +1890 +1891 +1892 +1893 +1894 +1895 +1896 +1897 +1898 +1899 +1900 +1901 +1902 +1903 +1904 +1905 +1906 +1907 +1908 +1909 +1910 +1911 +1912 +1913 +1914 +1915 +1916 +1917 +1918 +1919 +1920 +1921 +1922 +1923 +1924 +1925 +1926 +1927 +1928 +1929 +1930 +1931 +1932 +1933 +1934 +1935 +1936 +1937 +1938 +1939 +1940 +1941 +1942 +1944 +1945 +1946 +1947 +1948 +1949 +1950 +1951 +1952 +1953 + diff --git a/backend/corpora/gallica/tests/test_import.py b/backend/corpora/gallica/tests/test_import.py new file mode 100644 index 000000000..72ad4bb36 --- /dev/null +++ b/backend/corpora/gallica/tests/test_import.py @@ -0,0 +1,33 @@ +from datetime import datetime +import requests + +from conftest import mock_response +from addcorpus.python_corpora.load_corpus import load_corpus_definition + + +target_documents = [ + { + "content": "SOMMAIRE DE FIGARO PAGE 2. Les Cours, les Ambassades, le Monde et la Ville. Les Echos. La fin du Bulletin vert. 1929-1930. PAGE 3. La Dernière Heure. Avant la Conférence de La Haye. Les méfaits de la tempête. PAGE 4. La Vie sportive. Revue de la Presse. Anne Douglas Sedgwick Marthe Ludérac. PAGE 5. Henri Rebois L'Art espagnol à l'Exposition de Barcelone. Robert Brussel Le Mouvement musical. Guy de Passillé Les Etrennes. Jacques Patin Les Premières. Les Alguazils Courrier des Lettres. Marc Hélys Revues étrangères. PAGE 6. La Bourse La Cote des Valeurs. Le Programme des spectacles. PAGE 7. Courrier des théâtres. Les Courses LA POLITIQUE La diplomatie ", + "contributor": [ + "Villemessant, Hippolyte de (1810-1879). Directeur de publication", + "Jouvin, Benoît (1810-1886). Directeur de publication", + ], + "date": "1930-01-01", + "id": "bpt6k296099q", + "issue": "01 janvier 19301930/01/01 (Numéro 1).", + "url": "https://gallica.bnf.fr/ark:/12148/bpt6k296099q", + } +] + +def test_gallica_import(monkeypatch, gallica_corpus_settings): + monkeypatch.setattr(requests, "get", mock_response) + corpus_def = load_corpus_definition("figaro") + sources = corpus_def.sources( + start=datetime(year=1930, month=1, day=1), + end=datetime(year=1930, month=12, day=31), + ) + documents = list(corpus_def.documents(sources)) + assert len(documents) == 1 + for document, target in zip(documents, target_documents): + for target_key in target.keys(): + assert document.get(target_key) == target.get(target_key) diff --git a/backend/requirements.txt b/backend/requirements.txt index be911bd7c..3ac3fa29f 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -132,7 +132,7 @@ h11==0.14.0 # wsproto humanize==4.9.0 # via flower -ianalyzer-readers==0.2.1 +ianalyzer-readers==0.2.2 # via -r requirements.in idna==3.4 # via @@ -249,7 +249,9 @@ pygments==2.16.1 # rich # seleniumbase pyjwt[crypto]==2.8.0 - # via django-allauth + # via + # django-allauth + # pyjwt pynose==1.4.8 # via seleniumbase pyopenssl==23.2.0