From 671b0f6fc12cf2f85dd99b57aa19e1665fd2f4f2 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Wed, 24 Apr 2024 15:52:48 +0200 Subject: [PATCH 01/13] initial import --- e2e/pipelines/test_evaluation_pipeline.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 e2e/pipelines/test_evaluation_pipeline.py diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py new file mode 100644 index 0000000000..e69de29bb2 From abca308cfa2909cadc7411125ff50065383e2489 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Thu, 25 Apr 2024 12:17:59 +0200 Subject: [PATCH 02/13] wip --- e2e/pipelines/test_evaluation_pipeline.py | 234 ++++++++++++++++++ e2e/samples/test_documents/Annelid.txt | 35 +++ e2e/samples/test_documents/Culture.txt | 24 ++ e2e/samples/test_documents/Edmund_Burke.txt | 40 +++ e2e/samples/test_documents/Freemasonry.txt | 37 +++ e2e/samples/test_documents/John_Kerry.txt | 52 ++++ .../test_documents/Late_Middle_Ages.txt | 40 +++ e2e/samples/test_documents/Materialism.txt | 27 ++ e2e/samples/test_documents/Normans.txt | 45 ++++ e2e/samples/test_documents/Strasbourg.txt | 34 +++ e2e/samples/test_documents/War_on_Terror.txt | 31 +++ .../components/evaluators/document_mrr.py | 2 +- pyproject.toml | 2 +- 13 files changed, 601 insertions(+), 2 deletions(-) create mode 100644 e2e/samples/test_documents/Annelid.txt create mode 100644 e2e/samples/test_documents/Culture.txt create mode 100644 e2e/samples/test_documents/Edmund_Burke.txt create mode 100644 e2e/samples/test_documents/Freemasonry.txt create mode 100644 e2e/samples/test_documents/John_Kerry.txt create mode 100644 e2e/samples/test_documents/Late_Middle_Ages.txt create mode 100644 e2e/samples/test_documents/Materialism.txt create mode 100644 e2e/samples/test_documents/Normans.txt create mode 100644 e2e/samples/test_documents/Strasbourg.txt create mode 100644 e2e/samples/test_documents/War_on_Terror.txt diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py index e69de29bb2..36ee194b35 100644 --- a/e2e/pipelines/test_evaluation_pipeline.py +++ b/e2e/pipelines/test_evaluation_pipeline.py @@ -0,0 +1,234 @@ +import os +from typing import List + +from haystack import Document, Pipeline +from haystack.components.builders import AnswerBuilder, PromptBuilder +from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder +from haystack.components.evaluators import ( + DocumentMAPEvaluator, + DocumentMRREvaluator, + DocumentRecallEvaluator, + EvaluationResult, + FaithfulnessEvaluator, + SASEvaluator, +) +from haystack.components.evaluators.document_recall import RecallMode +from haystack.components.generators import OpenAIGenerator +from haystack.components.retrievers import InMemoryEmbeddingRetriever +from haystack.components.writers import DocumentWriter +from haystack.document_stores.in_memory import InMemoryDocumentStore +from haystack.document_stores.types import DuplicatePolicy + + +def indexing(documents: List[Document]): + """ + Indexing the documents + """ + document_store = InMemoryDocumentStore() + doc_writer = DocumentWriter(document_store=document_store, policy=DuplicatePolicy.SKIP) + doc_embedder = SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2") + ingestion_pipe = Pipeline() + ingestion_pipe.add_component(instance=doc_embedder, name="doc_embedder") + ingestion_pipe.add_component(instance=doc_writer, name="doc_writer") + ingestion_pipe.connect("doc_embedder.documents", "doc_writer.documents") + ingestion_pipe.run({"doc_embedder": {"documents": documents}}) + return document_store + + +def build_rag_pipeline(document_store, top_k=2): + """ + Building the RAG pipeline + """ + template = """ + You have to answer the following question based on the given context information only. + + Context: + {% for document in documents %} + {{ document.content }} + {% endfor %} + + Question: {{question}} + Answer: + """ + + rag_pipeline_1 = Pipeline() + rag_pipeline_1.add_component( + "query_embedder", SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2") + ) + rag_pipeline_1.add_component("retriever", InMemoryEmbeddingRetriever(document_store, top_k=top_k)) + rag_pipeline_1.add_component("prompt_builder", PromptBuilder(template=template)) + rag_pipeline_1.add_component("generator", OpenAIGenerator(model="gpt-3.5-turbo")) + rag_pipeline_1.add_component("answer_builder", AnswerBuilder()) + + rag_pipeline_1.connect("query_embedder", "retriever.query_embedding") + rag_pipeline_1.connect("retriever", "prompt_builder.documents") + rag_pipeline_1.connect("prompt_builder", "generator") + rag_pipeline_1.connect("generator.replies", "answer_builder.replies") + rag_pipeline_1.connect("generator.meta", "answer_builder.meta") + rag_pipeline_1.connect("retriever", "answer_builder.documents") + + return rag_pipeline_1 + + +def test_evaluation_pipeline(samples_path): + """ + Test the evaluation pipeline + """ + documents = [] + + def create_document(text: str, name: str): + return Document(content=text, meta={"name": name}) + + for root, dirs, files in os.walk(str(samples_path) + "/test_documents/"): + for article in files: + with open(f"{root}/{article}", "r") as f: + raw_texts = f.read().split("\n") + for text in raw_texts: + documents.append(create_document(text, article)) + + document_store = indexing(documents) + + # collect all the data for evaluation + all_questions = [] + all_ground_truth_documents = [] + all_ground_truth_answers = [] + all_retrieved_documents = [] + all_contexts = [] + all_answers = [] + + questions = [ + { + "question": "Who re-translated the Reflections into French?", + "answer": ["Louis XVI"], + "ground_truth_doc": ["Edmund_Burke.txt"], + }, + { + "question": "What was Kerry's role in the Yale Political Union as a junior?", + "answer": ["President of the Union"], + "ground_truth_doc": ["John_Kerry.txt"], + }, + { + "question": 'What falls within the term "cultural anthropology"?', + "answer": ["the ideology and analytical stance of cultural relativism"], + "ground_truth_doc": ["Culture.txt"], + }, + { + "question": "Who was the spiritual guide during the Protestant Reformation?", + "answer": ["Martin Bucer"], + "ground_truth_doc": ["Strasbourg.txt"], + }, + { + "question": "What separates many annelids' segments?", + "answer": ["Septa"], + "ground_truth_doc": ["Annelid.txt"], + }, + { + "question": "What is materialism?", + "answer": ["a form of philosophical monism"], + "ground_truth_doc": ["Materialism.txt"], + }, + { + "question": "Who did the Hungarian nobility elect as King of Hungary?", + "answer": ["Matthias"], + "ground_truth_doc": ["Late_Middle_Ages.txt"], + }, + ] + + rag_pipeline_1 = build_rag_pipeline(document_store, top_k=2) + + # ToDo: do this in batch to avoid multiple calls to the pipeline + for q in questions: + question = q["question"] + answer = q["answer"] + ground_truth_docs = [doc for doc in documents if doc.meta["name"] in q["ground_truth_doc"]] + all_ground_truth_documents.append(ground_truth_docs) + all_ground_truth_answers.append(answer[0]) + all_questions.append(question) + + response = rag_pipeline_1.run( + { + "query_embedder": {"text": question}, + "prompt_builder": {"question": question}, + "answer_builder": {"query": question}, + } + ) + + all_retrieved_documents.append(response["answer_builder"]["answers"][0].documents) + all_contexts.append([doc.content for doc in response["answer_builder"]["answers"][0].documents]) + all_answers.append(response["answer_builder"]["answers"][0].data) + + eval_pipeline = Pipeline() + eval_pipeline.add_component("doc_mrr", DocumentMRREvaluator()) + eval_pipeline.add_component("groundness", FaithfulnessEvaluator()) + eval_pipeline.add_component("sas", SASEvaluator(model="sentence-transformers/all-MiniLM-L6-v2")) + eval_pipeline.add_component("doc_map", DocumentMAPEvaluator()) + eval_pipeline.add_component("doc_recall_single_hit", DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT)) + eval_pipeline.add_component("doc_recall_multi_hit", DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT)) + + results = eval_pipeline.run( + { + "doc_mrr": { + "ground_truth_documents": all_ground_truth_documents, + "retrieved_documents": all_retrieved_documents, + }, + "groundness": {"questions": all_questions, "contexts": all_contexts, "responses": all_answers}, + "sas": {"predicted_answers": all_answers, "ground_truth_answers": all_ground_truth_answers}, + "doc_map": { + "ground_truth_documents": all_ground_truth_documents, + "retrieved_documents": all_retrieved_documents, + }, + "doc_recall_single_hit": { + "ground_truth_documents": all_ground_truth_documents, + "retrieved_documents": all_retrieved_documents, + }, + "doc_recall_multi_hit": { + "ground_truth_documents": all_ground_truth_documents, + "retrieved_documents": all_retrieved_documents, + }, + } + ) + + data = { + "inputs": { + "question": all_questions, + "contexts": all_contexts, + "answer": all_ground_truth_answers, + "predicted_answer": all_answers, + }, + "metrics": [ + { + "name": "Mean Reciprocal Rank", + "individual_scores": results["doc_mrr"]["individual_scores"], + "score": results["doc_mrr"]["score"], + }, + { + "name": "Semantic Answer Similarity", + "individual_scores": results["sas"]["individual_scores"], + "score": results["sas"]["score"], + }, + { + "name": "Faithfulness", + "individual_scores": results["groundness"]["individual_scores"], + "score": results["groundness"]["score"], + }, + { + "name": "Document MAP", + "individual_scores": results["doc_map"]["individual_scores"], + "score": results["doc_map"]["score"], + }, + { + "name": "Document Recall Single Hit", + "individual_scores": results["doc_recall_single_hit"]["individual_scores"], + "score": results["doc_recall_single_hit"]["score"], + }, + { + "name": "Document Recall Multi Hit", + "individual_scores": results["doc_recall_multi_hit"]["individual_scores"], + "score": results["doc_recall_multi_hit"]["score"], + }, + ], + } + + evaluation_result = EvaluationResult(pipeline_name="pipe_1", results=data) + + print(evaluation_result) diff --git a/e2e/samples/test_documents/Annelid.txt b/e2e/samples/test_documents/Annelid.txt new file mode 100644 index 0000000000..8cf40a1bb4 --- /dev/null +++ b/e2e/samples/test_documents/Annelid.txt @@ -0,0 +1,35 @@ +The annelids are bilaterally symmetrical, triploblastic, coelomate, invertebrate organisms. They also have parapodia for locomotion. Most textbooks still use the traditional division into polychaetes (almost all marine), oligochaetes (which include earthworms) and leech-like species. Cladistic research since 1997 has radically changed this scheme, viewing leeches as a sub-group of oligochaetes and oligochaetes as a sub-group of polychaetes. In addition, the Pogonophora, Echiura and Sipuncula, previously regarded as separate phyla, are now regarded as sub-groups of polychaetes. Annelids are considered members of the Lophotrochozoa, a "super-phylum" of protostomes that also includes molluscs, brachiopods, flatworms and nemerteans. +The basic annelid form consists of multiple segments. Each segment has the same sets of organs and, in most polychaetes, has a pair of parapodia that many species use for locomotion. Septa separate the segments of many species, but are poorly defined or absent in others, and Echiura and Sipuncula show no obvious signs of segmentation. In species with well-developed septa, the blood circulates entirely within blood vessels, and the vessels in segments near the front ends of these species are often built up with muscles that act as hearts. The septa of such species also enable them to change the shapes of individual segments, which facilitates movement by peristalsis ("ripples" that pass along the body) or by undulations that improve the effectiveness of the parapodia. In species with incomplete septa or none, the blood circulates through the main body cavity without any kind of pump, and there is a wide range of locomotory techniques – some burrowing species turn their pharynges inside out to drag themselves through the sediment. +Although many species can reproduce asexually and use similar mechanisms to regenerate after severe injuries, sexual reproduction is the normal method in species whose reproduction has been studied. The minority of living polychaetes whose reproduction and lifecycles are known produce trochophore larvae, that live as plankton and then sink and metamorphose into miniature adults. Oligochaetes are full hermaphrodites and produce a ring-like cocoon around their bodies, in which the eggs and hatchlings are nourished until they are ready to emerge. +Earthworms are Oligochaetes that support terrestrial food chains both as prey and in some regions are important in aeration and enriching of soil. The burrowing of marine polychaetes, which may constitute up to a third of all species in near-shore environments, encourages the development of ecosystems by enabling water and oxygen to penetrate the sea floor. In addition to improving soil fertility, annelids serve humans as food and as bait. Scientists observe annelids to monitor the quality of marine and fresh water. Although blood-letting is no longer in favor with doctors, some leech species are regarded as endangered species because they have been over-harvested for this purpose in the last few centuries. Ragworms' jaws are now being studied by engineers as they offer an exceptional combination of lightness and strength. +Since annelids are soft-bodied, their fossils are rare – mostly jaws and the mineralized tubes that some of the species secreted. Although some late Ediacaran fossils may represent annelids, the oldest known fossil that is identified with confidence comes from about 518 million years ago in the early Cambrian period. Fossils of most modern mobile polychaete groups appeared by the end of the Carboniferous, about 299 million years ago. Palaeontologists disagree about whether some body fossils from the mid Ordovician, about 472 to 461 million years ago, are the remains of oligochaetes, and the earliest indisputable fossils of the group appear in the Tertiary period, which began 65 million years ago. +No single feature distinguishes Annelids from other invertebrate phyla, but they have a distinctive combination of features. Their bodies are long, with segments that are divided externally by shallow ring-like constrictions called annuli and internally by septa ("partitions") at the same points, although in some species the septa are incomplete and in a few cases missing. Most of the segments contain the same sets of organs, although sharing a common gut, circulatory system and nervous system makes them inter-dependent. Their bodies are covered by a cuticle (outer covering) that does not contain cells but is secreted by cells in the skin underneath, is made of tough but flexible collagen and does not molt – on the other hand arthropods' cuticles are made of the more rigid α-chitin, and molt until the arthropods reach their full size. Most annelids have closed circulatory systems, where the blood makes its entire circuit via blood vessels. +Most of an annelid's body consists of segments that are practically identical, having the same sets of internal organs and external chaetae (Greek χαιτη, meaning "hair") and, in some species, appendages. However, the frontmost and rearmost sections are not regarded as true segments as they do not contain the standard sets of organs and do not develop in the same way as the true segments. The frontmost section, called the prostomium (Greek προ- meaning "in front of" and στομα meaning "mouth") contains the brain and sense organs, while the rearmost, called the pygidium (Greek πυγιδιον, meaning "little tail") or periproct contains the anus, generally on the underside. The first section behind the prostomium, called the peristomium (Greek περι- meaning "around" and στομα meaning "mouth"), is regarded by some zoologists as not a true segment, but in some polychaetes the peristomium has chetae and appendages like those of other segments. +Annelids' cuticles are made of collagen fibers, usually in layers that spiral in alternating directions so that the fibers cross each other. These are secreted by the one-cell deep epidermis (outermost skin layer). A few marine annelids that live in tubes lack cuticles, but their tubes have a similar structure, and mucus-secreting glands in the epidermis protect their skins. Under the epidermis is the dermis, which is made of connective tissue, in other words a combination of cells and non-cellular materials such as collagen. Below this are two layers of muscles, which develop from the lining of the coelom (body cavity): circular muscles make a segment longer and slimmer when they contract, while under them are longitudinal muscles, usually four distinct strips, whose contractions make the segment shorter and fatter. Some annelids also have oblique internal muscles that connect the underside of the body to each side. +The setae ("hairs") of annelids project out from the epidermis to provide traction and other capabilities. The simplest are unjointed and form paired bundles near the top and bottom of each side of each segment. The parapodia ("limbs") of annelids that have them often bear more complex chetae at their tips – for example jointed, comb-like or hooked. Chetae are made of moderately flexible β-chitin and are formed by follicles, each of which has a chetoblast ("hair-forming") cell at the bottom and muscles that can extend or retract the cheta. The chetoblasts produce chetae by forming microvilli, fine hair-like extensions that increase the area available for secreting the cheta. When the cheta is complete, the microvilli withdraw into the chetoblast, leaving parallel tunnels that run almost the full length of the cheta. Hence annelids' chetae are structurally different from the setae ("bristles") of arthropods, which are made of the more rigid α-chitin, have a single internal cavity, and are mounted on flexible joints in shallow pits in the cuticle. +Nearly all polychaetes have parapodia that function as limbs, while other major annelid groups lack them. Parapodia are unjointed paired extensions of the body wall, and their muscles are derived from the circular muscles of the body. They are often supported internally by one or more large, thick chetae. The parapodia of burrowing and tube-dwelling polychaetes are often just ridges whose tips bear hooked chetae. In active crawlers and swimmers the parapodia are often divided into large upper and lower paddles on a very short trunk, and the paddles are generally fringed with chetae and sometimes with cirri (fused bundles of cilia) and gills. +The brain generally forms a ring round the pharynx (throat), consisting of a pair of ganglia (local control centers) above and in front of the pharynx, linked by nerve cords either side of the pharynx to another pair of ganglia just below and behind it. The brains of polychaetes are generally in the prostomium, while those of clitellates are in the peristomium or sometimes the first segment behind the peristomium. In some very mobile and active polychaetes the brain is enlarged and more complex, with visible hindbrain, midbrain and forebrain sections. The rest of the central nervous system is generally "ladder-like", consisting of a pair of nerve cords that run through the bottom part of the body and have in each segment paired ganglia linked by a transverse connection. From each segmental ganglion a branching system of local nerves runs into the body wall and then encircles the body. However, in most polychaetes the two main nerve cords are fused, and in the tube-dwelling genus Owenia the single nerve chord has no ganglia and is located in the epidermis. +As in arthropods, each muscle fiber (cell) is controlled by more than one neuron, and the speed and power of the fiber's contractions depends on the combined effects of all its neurons. Vertebrates have a different system, in which one neuron controls a group of muscle fibers. Most annelids' longitudinal nerve trunks include giant axons (the output signal lines of nerve cells). Their large diameter decreases their resistance, which allows them to transmit signals exceptionally fast. This enables these worms to withdraw rapidly from danger by shortening their bodies. Experiments have shown that cutting the giant axons prevents this escape response but does not affect normal movement. +The sensors are primarily single cells that detect light, chemicals, pressure waves and contact, and are present on the head, appendages (if any) and other parts of the body. Nuchal ("on the neck") organs are paired, ciliated structures found only in polychaetes, and are thought to be chemosensors. Some polychaetes also have various combinations of ocelli ("little eyes") that detect the direction from which light is coming and camera eyes or compound eyes that can probably form images. The compound eyes probably evolved independently of arthropods' eyes. Some tube-worms use ocelli widely spread over their bodies to detect the shadows of fish, so that they can quickly withdraw into their tubes. Some burrowing and tube-dwelling polychaetes have statocysts (tilt and balance sensors) that tell them which way is down. A few polychaete genera have on the undersides of their heads palps that are used both in feeding and as "feelers", and some of these also have antennae that are structurally similar but probably are used mainly as "feelers". +Most annelids have a pair of coelomata (body cavities) in each segment, separated from other segments by septa and from each other by vertical mesenteries. Each septum forms a sandwich with connective tissue in the middle and mesothelium (membrane that serves as a lining) from the preceding and following segments on either side. Each mesentery is similar except that the mesothelium is the lining of each of the pair of coelomata, and the blood vessels and, in polychaetes, the main nerve cords are embedded in it. The mesothelium is made of modified epitheliomuscular cells; in other words, their bodies form part of the epithelium but their bases extend to form muscle fibers in the body wall. The mesothelium may also form radial and circular muscles on the septa, and circular muscles around the blood vessels and gut. Parts of the mesothelium, especially on the outside of the gut, may also form chloragogen cells that perform similar functions to the livers of vertebrates: producing and storing glycogen and fat; producing the oxygen-carrier hemoglobin; breaking down proteins; and turning nitrogenous waste products into ammonia and urea to be excreted. +Many annelids move by peristalsis (waves of contraction and expansion that sweep along the body), or flex the body while using parapodia to crawl or swim. In these animals the septa enable the circular and longitudinal muscles to change the shape of individual segments, by making each segment a separate fluid-filled "balloon". However, the septa are often incomplete in annelids that are semi-sessile or that do not move by peristalsis or by movements of parapodia – for example some move by whipping movements of the body, some small marine species move by means of cilia (fine muscle-powered hairs) and some burrowers turn their pharynges (throats) inside out to penetrate the sea-floor and drag themselves into it. +The fluid in the coelomata contains coelomocyte cells that defend the animals against parasites and infections. In some species coelomocytes may also contain a respiratory pigment – red hemoglobin in some species, green chlorocruorin in others (dissolved in the plasma) – and provide oxygen transport within their segments. Respiratory pigment is also dissolved in the blood plasma. Species with well-developed septa generally also have blood vessels running all long their bodies above and below the gut, the upper one carrying blood forwards while the lower one carries it backwards. Networks of capillaries in the body wall and around the gut transfer blood between the main blood vessels and to parts of the segment that need oxygen and nutrients. Both of the major vessels, especially the upper one, can pump blood by contracting. In some annelids the forward end of the upper blood vessel is enlarged with muscles to form a heart, while in the forward ends of many earthworms some of the vessels that connect the upper and lower main vessels function as hearts. Species with poorly developed or no septa generally have no blood vessels and rely on the circulation within the coelom for delivering nutrients and oxygen. +However, leeches and their closest relatives have a body structure that is very uniform within the group but significantly different from that of other annelids, including other members of the Clitellata. In leeches there are no septa, the connective tissue layer of the body wall is so thick that it occupies much of the body, and the two coelomata are widely separated and run the length of the body. They function as the main blood vessels, although they are side-by-side rather than upper and lower. However, they are lined with mesothelium, like the coelomata and unlike the blood vessels of other annelids. Leeches generally use suckers at their front and rear ends to move like inchworms. The anus is on the upper surface of the pygidium. +Feeding structures in the mouth region vary widely, and have little correlation with the animals' diets. Many polychaetes have a muscular pharynx that can be everted (turned inside out to extend it). In these animals the foremost few segments often lack septa so that, when the muscles in these segments contract, the sharp increase in fluid pressure from all these segments everts the pharynx very quickly. Two families, the Eunicidae and Phyllodocidae, have evolved jaws, which can be used for seizing prey, biting off pieces of vegetation, or grasping dead and decaying matter. On the other hand, some predatory polychaetes have neither jaws nor eversible pharynges. Selective deposit feeders generally live in tubes on the sea-floor and use palps to find food particles in the sediment and then wipe them into their mouths. Filter feeders use "crowns" of palps covered in cilia that wash food particles towards their mouths. Non-selective deposit feeders ingest soil or marine sediments via mouths that are generally unspecialized. Some clitellates have sticky pads in the roofs of their mouths, and some of these can evert the pads to capture prey. Leeches often have an eversible proboscis, or a muscular pharynx with two or three teeth. +The gut is generally an almost straight tube supported by the mesenteries (vertical partitions within segments), and ends with the anus on the underside of the pygidium. However, in members of the tube-dwelling family Siboglinidae the gut is blocked by a swollen lining that houses symbiotic bacteria, which can make up 15% of the worms' total weight. The bacteria convert inorganic matter – such as hydrogen sulfide and carbon dioxide from hydrothermal vents, or methane from seeps – to organic matter that feeds themselves and their hosts, while the worms extend their palps into the gas flows to absorb the gases needed by the bacteria. +Annelids with blood vessels use metanephridia to remove soluble waste products, while those without use protonephridia. Both of these systems use a two-stage filtration process, in which fluid and waste products are first extracted and these are filtered again to re-absorb any re-usable materials while dumping toxic and spent materials as urine. The difference is that protonephridia combine both filtration stages in the same organ, while metanephridia perform only the second filtration and rely on other mechanisms for the first – in annelids special filter cells in the walls of the blood vessels let fluids and other small molecules pass into the coelomic fluid, where it circulates to the metanephridia. In annelids the points at which fluid enters the protonephridia or metanephridia are on the forward side of a septum while the second-stage filter and the nephridiopore (exit opening in the body wall) are in the following segment. As a result, the hindmost segment (before the growth zone and pygidium) has no structure that extracts its wastes, as there is no following segment to filter and discharge them, while the first segment contains an extraction structure that passes wastes to the second, but does not contain the structures that re-filter and discharge urine. +It is thought that annelids were originally animals with two separate sexes, which released ova and sperm into the water via their nephridia. The fertilized eggs develop into trochophore larvae, which live as plankton. Later they sink to the sea-floor and metamorphose into miniature adults: the part of the trochophore between the apical tuft and the prototroch becomes the prostomium (head); a small area round the trochophore's anus becomes the pygidium (tail-piece); a narrow band immediately in front of that becomes the growth zone that produces new segments; and the rest of the trochophore becomes the peristomium (the segment that contains the mouth). +However, the lifecycles of most living polychaetes, which are almost all marine animals, are unknown, and only about 25% of the 300+ species whose lifecycles are known follow this pattern. About 14% use a similar external fertilization but produce yolk-rich eggs, which reduce the time the larva needs to spend among the plankton, or eggs from which miniature adults emerge rather than larvae. The rest care for the fertilized eggs until they hatch – some by producing jelly-covered masses of eggs which they tend, some by attaching the eggs to their bodies and a few species by keeping the eggs within their bodies until they hatch. These species use a variety of methods for sperm transfer; for example, in some the females collect sperm released into the water, while in others the males have a penis that inject sperm into the female. There is no guarantee that this is a representative sample of polychaetes' reproductive patterns, and it simply reflects scientists' current knowledge. +Some polychaetes breed only once in their lives, while others breed almost continuously or through several breeding seasons. While most polychaetes remain of one sex all their lives, a significant percentage of species are full hermaphrodites or change sex during their lives. Most polychaetes whose reproduction has been studied lack permanent gonads, and it is uncertain how they produce ova and sperm. In a few species the rear of the body splits off and becomes a separate individual that lives just long enough to swim to a suitable environment, usually near the surface, and spawn. +Most mature clitellates (the group that includes earthworms and leeches) are full hermaphrodites, although in a few leech species younger adults function as males and become female at maturity. All have well-developed gonads, and all copulate. Earthworms store their partners' sperm in spermathecae ("sperm stores") and then the clitellum produces a cocoon that collects ova from the ovaries and then sperm from the spermathecae. Fertilization and development of earthworm eggs takes place in the cocoon. Leeches' eggs are fertilized in the ovaries, and then transferred to the cocoon. In all clitellates the cocoon also either produces yolk when the eggs are fertilized or nutrients while they are developing. All clitellates hatch as miniature adults rather than larvae. +Charles Darwin's book The Formation of Vegetable Mould through the Action of Worms (1881) presented the first scientific analysis of earthworms' contributions to soil fertility. Some burrow while others live entirely on the surface, generally in moist leaf litter. The burrowers loosen the soil so that oxygen and water can penetrate it, and both surface and burrowing worms help to produce soil by mixing organic and mineral matter, by accelerating the decomposition of organic matter and thus making it more quickly available to other organisms, and by concentrating minerals and converting them to forms that plants can use more easily. Earthworms are also important prey for birds ranging in size from robins to storks, and for mammals ranging from shrews to badgers, and in some cases conserving earthworms may be essential for conserving endangered birds. +Terrestrial annelids can be invasive in some situations. In the glaciated areas of North America, for example, almost all native earthworms are thought to have been killed by the glaciers and the worms currently found in those areas are all introduced from other areas, primarily from Europe, and, more recently, from Asia. Northern hardwood forests are especially negatively impacted by invasive worms through the loss of leaf duff, soil fertility, changes in soil chemistry and the loss of ecological diversity. Especially of concern is Amynthas agrestis and at least one state (Wisconsin) has listed it as a prohibited species. +Earthworms make a significant contribution to soil fertility. The rear end of the Palolo worm, a marine polychaete that tunnels through coral, detaches in order to spawn at the surface, and the people of Samoa regard these spawning modules as a delicacy. Anglers sometimes find that worms are more effective bait than artificial flies, and worms can be kept for several days in a tin lined with damp moss. Ragworms are commercially important as bait and as food sources for aquaculture, and there have been proposals to farm them in order to reduce over-fishing of their natural populations. Some marine polychaetes' predation on molluscs causes serious losses to fishery and aquaculture operations. +Accounts of the use of leeches for the medically dubious practise of blood-letting have come from China around 30 AD, India around 200 AD, ancient Rome around 50 AD and later throughout Europe. In the 19th century medical demand for leeches was so high that some areas' stocks were exhausted and other regions imposed restrictions or bans on exports, and Hirudo medicinalis is treated as an endangered species by both IUCN and CITES. More recently leeches have been used to assist in microsurgery, and their saliva has provided anti-inflammatory compounds and several important anticoagulants, one of which also prevents tumors from spreading. +Since annelids are soft-bodied, their fossils are rare. Polychaetes' fossil record consists mainly of the jaws that some species had and the mineralized tubes that some secreted. Some Ediacaran fossils such as Dickinsonia in some ways resemble polychaetes, but the similarities are too vague for these fossils to be classified with confidence. The small shelly fossil Cloudina, from 549 to 542 million years ago, has been classified by some authors as an annelid, but by others as a cnidarian (i.e. in the phylum to which jellyfish and sea anemones belong). Until 2008 the earliest fossils widely accepted as annelids were the polychaetes Canadia and Burgessochaeta, both from Canada's Burgess Shale, formed about 505 million years ago in the early Cambrian. Myoscolex, found in Australia and a little older than the Burgess Shale, was possibly an annelid. However, it lacks some typical annelid features and has features which are not usually found in annelids and some of which are associated with other phyla. Then Simon Conway Morris and John Peel reported Phragmochaeta from Sirius Passet, about 518 million years old, and concluded that it was the oldest annelid known to date. There has been vigorous debate about whether the Burgess Shale fossil Wiwaxia was a mollusc or an annelid. Polychaetes diversified in the early Ordovician, about 488 to 474 million years ago. It is not until the early Ordovician that the first annelid jaws are found, thus the crown-group cannot have appeared before this date and probably appeared somewhat later. By the end of the Carboniferous, about 299 million years ago, fossils of most of the modern mobile polychaete groups had appeared. Many fossil tubes look like those made by modern sessile polychaetes , but the first tubes clearly produced by polychaetes date from the Jurassic, less than 199 million years ago. +The earliest good evidence for oligochaetes occurs in the Tertiary period, which began 65 million years ago, and it has been suggested that these animals evolved around the same time as flowering plants in the early Cretaceous, from 130 to 90 million years ago. A trace fossil consisting of a convoluted burrow partly filled with small fecal pellets may be evidence that earthworms were present in the early Triassic period from 251 to 245 million years ago. Body fossils going back to the mid Ordovician, from 472 to 461 million years ago, have been tentatively classified as oligochaetes, but these identifications are uncertain and some have been disputed. +Traditionally the annelids have been divided into two major groups, the polychaetes and clitellates. In turn the clitellates were divided into oligochaetes, which include earthworms, and hirudinomorphs, whose best-known members are leeches. For many years there was no clear arrangement of the approximately 80 polychaete families into higher-level groups. In 1997 Greg Rouse and Kristian Fauchald attempted a "first heuristic step in terms of bringing polychaete systematics to an acceptable level of rigour", based on anatomical structures, and divided polychaetes into: +In 2007 Torsten Struck and colleagues compared 3 genes in 81 taxa, of which 9 were outgroups, in other words not considered closely related to annelids but included to give an indication of where the organisms under study are placed on the larger tree of life. For a cross-check the study used an analysis of 11 genes (including the original 3) in 10 taxa. This analysis agreed that clitellates, pogonophorans and echiurans were on various branches of the polychaete family tree. It also concluded that the classification of polychaetes into Scolecida, Canalipalpata and Aciculata was useless, as the members of these alleged groups were scattered all over the family tree derived from comparing the 81 taxa. In addition, it also placed sipunculans, generally regarded at the time as a separate phylum, on another branch of the polychaete tree, and concluded that leeches were a sub-group of oligochaetes rather than their sister-group among the clitellates. Rouse accepted the analyses based on molecular phylogenetics, and their main conclusions are now the scientific consensus, although the details of the annelid family tree remain uncertain. +In addition to re-writing the classification of annelids and 3 previously independent phyla, the molecular phylogenetics analyses undermine the emphasis that decades of previous writings placed on the importance of segmentation in the classification of invertebrates. Polychaetes, which these analyses found to be the parent group, have completely segmented bodies, while polychaetes' echiurans and sipunculan offshoots are not segmented and pogonophores are segmented only in the rear parts of their bodies. It now seems that segmentation can appear and disappear much more easily in the course of evolution than was previously thought. The 2007 study also noted that the ladder-like nervous system, which is associated with segmentation, is less universal previously thought in both annelids and arthropods.[n 2] +Annelids are members of the protostomes, one of the two major superphyla of bilaterian animals – the other is the deuterostomes, which includes vertebrates. Within the protostomes, annelids used to be grouped with arthropods under the super-group Articulata ("jointed animals"), as segmentation is obvious in most members of both phyla. However, the genes that drive segmentation in arthropods do not appear to do the same in annelids. Arthropods and annelids both have close relatives that are unsegmented. It is at least as easy to assume that they evolved segmented bodies independently as it is to assume that the ancestral protostome or bilaterian was segmented and that segmentation disappeared in many descendant phyla. The current view is that annelids are grouped with molluscs, brachiopods and several other phyla that have lophophores (fan-like feeding structures) and/or trochophore larvae as members of Lophotrochozoa. Bryzoa may be the most basal phylum (the one that first became distinctive) within the Lophotrochozoa, and the relationships between the other members are not yet known. Arthropods are now regarded as members of the Ecdysozoa ("animals that molt"), along with some phyla that are unsegmented. +The "Lophotrochozoa" hypothesis is also supported by the fact that many phyla within this group, including annelids, molluscs, nemerteans and flatworms, follow a similar pattern in the fertilized egg's development. When their cells divide after the 4-cell stage, descendants of these 4 cells form a spiral pattern. In these phyla the "fates" of the embryo's cells, in other words the roles their descendants will play in the adult animal, are the same and can be predicted from a very early stage. Hence this development pattern is often described as "spiral determinate cleavage". diff --git a/e2e/samples/test_documents/Culture.txt b/e2e/samples/test_documents/Culture.txt new file mode 100644 index 0000000000..64843b52c0 --- /dev/null +++ b/e2e/samples/test_documents/Culture.txt @@ -0,0 +1,24 @@ +Cambridge English Dictionary states that culture is, "the way of life, especially the general customs and beliefs, of a particular group of people at a particular time." Terror Management Theory posits that culture is a series of activities and worldviews that provide humans with the illusion of being individuals of value in a world meaning—raising themselves above the merely physical aspects of existence, in order to deny the animal insignificance and death that Homo Sapiens became aware of when they acquired a larger brain. +As a defining aspect of what it means to be human, culture is a central concept in anthropology, encompassing the range of phenomena that are transmitted through social learning in human societies. The word is used in a general sense as the evolved ability to categorize and represent experiences with symbols and to act imaginatively and creatively. This ability arose with the evolution of behavioral modernity in humans around 50,000 years ago.[citation needed] This capacity is often thought to be unique to humans, although some other species have demonstrated similar, though much less complex abilities for social learning. It is also used to denote the complex networks of practices and accumulated knowledge and ideas that is transmitted through social interaction and exist in specific human groups, or cultures, using the plural form. Some aspects of human behavior, such as language, social practices such as kinship, gender and marriage, expressive forms such as art, music, dance, ritual, religion, and technologies such as cooking, shelter, clothing are said to be cultural universals, found in all human societies. The concept material culture covers the physical expressions of culture, such as technology, architecture and art, whereas the immaterial aspects of culture such as principles of social organization (including, practices of political organization and social institutions), mythology, philosophy, literature (both written and oral), and science make up the intangible cultural heritage of a society. +In the humanities, one sense of culture, as an attribute of the individual, has been the degree to which they have cultivated a particular level of sophistication, in the arts, sciences, education, or manners. The level of cultural sophistication has also sometimes been seen to distinguish civilizations from less complex societies. Such hierarchical perspectives on culture are also found in class-based distinctions between a high culture of the social elite and a low culture, popular culture or folk culture of the lower classes, distinguished by the stratified access to cultural capital. In common parlance, culture is often used to refer specifically to the symbolic markers used by ethnic groups to distinguish themselves visibly from each other such as body modification, clothing or jewelry.[dubious – discuss] Mass culture refers to the mass-produced and mass mediated forms of consumer culture that emerged in the 20th century. Some schools of philosophy, such as Marxism and critical theory, have argued that culture is often used politically as a tool of the elites to manipulate the lower classes and create a false consciousness, such perspectives common in the discipline of cultural studies. In the wider social sciences, the theoretical perspective of cultural materialism holds that human symbolic culture arises from the material conditions of human life, as humans create the conditions for physical survival, and that the basis of culture is found in evolved biological dispositions. +When used as a count noun "a culture", is the set of customs, traditions and values of a society or community, such as an ethnic group or nation. In this sense, multiculturalism is a concept that values the peaceful coexistence and mutual respect between different cultures inhabiting the same territory. Sometimes "culture" is also used to describe specific practices within a subgroup of a society, a subculture (e.g. "bro culture"), or a counter culture. Within cultural anthropology, the ideology and analytical stance of cultural relativism holds that cultures cannot easily be objectively ranked or evaluated because any evaluation is necessarily situated within the value system of a given culture. +The modern term "culture" is based on a term used by the Ancient Roman orator Cicero in his Tusculanae Disputationes, where he wrote of a cultivation of the soul or "cultura animi", using an agricultural metaphor for the development of a philosophical soul, understood teleologically as the highest possible ideal for human development. Samuel Pufendorf took over this metaphor in a modern context, meaning something similar, but no longer assuming that philosophy was man's natural perfection. His use, and that of many writers after him "refers to all the ways in which human beings overcome their original barbarism, and through artifice, become fully human". +Social conflict and the development of technologies can produce changes within a society by altering social dynamics and promoting new cultural models, and spurring or enabling generative action. These social shifts may accompany ideological shifts and other types of cultural change. For example, the U.S. feminist movement involved new practices that produced a shift in gender relations, altering both gender and economic structures. Environmental conditions may also enter as factors. For example, after tropical forests returned at the end of the last ice age, plants suitable for domestication were available, leading to the invention of agriculture, which in turn brought about many cultural innovations and shifts in social dynamics. +Cultures are externally affected via contact between societies, which may also produce—or inhibit—social shifts and changes in cultural practices. War or competition over resources may impact technological development or social dynamics. Additionally, cultural ideas may transfer from one society to another, through diffusion or acculturation. In diffusion, the form of something (though not necessarily its meaning) moves from one culture to another. For example, hamburgers, fast food in the United States, seemed exotic when introduced into China. "Stimulus diffusion" (the sharing of ideas) refers to an element of one culture leading to an invention or propagation in another. "Direct Borrowing" on the other hand tends to refer to technological or tangible diffusion from one culture to another. Diffusion of innovations theory presents a research-based model of why and when individuals and cultures adopt new ideas, practices, and products. +Immanuel Kant (1724–1804) has formulated an individualist definition of "enlightenment" similar to the concept of bildung: "Enlightenment is man's emergence from his self-incurred immaturity." He argued that this immaturity comes not from a lack of understanding, but from a lack of courage to think independently. Against this intellectual cowardice, Kant urged: Sapere aude, "Dare to be wise!" In reaction to Kant, German scholars such as Johann Gottfried Herder (1744–1803) argued that human creativity, which necessarily takes unpredictable and highly diverse forms, is as important as human rationality. Moreover, Herder proposed a collective form of bildung: "For Herder, Bildung was the totality of experiences that provide a coherent identity, and sense of common destiny, to a people." +In 1795, the Prussian linguist and philosopher Wilhelm von Humboldt (1767–1835) called for an anthropology that would synthesize Kant's and Herder's interests. During the Romantic era, scholars in Germany, especially those concerned with nationalist movements—such as the nationalist struggle to create a "Germany" out of diverse principalities, and the nationalist struggles by ethnic minorities against the Austro-Hungarian Empire—developed a more inclusive notion of culture as "worldview" (Weltanschauung). According to this school of thought, each ethnic group has a distinct worldview that is incommensurable with the worldviews of other groups. Although more inclusive than earlier views, this approach to culture still allowed for distinctions between "civilized" and "primitive" or "tribal" cultures. +In 1860, Adolf Bastian (1826–1905) argued for "the psychic unity of mankind". He proposed that a scientific comparison of all human societies would reveal that distinct worldviews consisted of the same basic elements. According to Bastian, all human societies share a set of "elementary ideas" (Elementargedanken); different cultures, or different "folk ideas" (Völkergedanken), are local modifications of the elementary ideas. This view paved the way for the modern understanding of culture. Franz Boas (1858–1942) was trained in this tradition, and he brought it with him when he left Germany for the United States. +In practice, culture referred to an élite ideal and was associated with such activities as art, classical music, and haute cuisine. As these forms were associated with urban life, "culture" was identified with "civilization" (from lat. civitas, city). Another facet of the Romantic movement was an interest in folklore, which led to identifying a "culture" among non-elites. This distinction is often characterized as that between high culture, namely that of the ruling social group, and low culture. In other words, the idea of "culture" that developed in Europe during the 18th and early 19th centuries reflected inequalities within European societies. +Matthew Arnold contrasted "culture" with anarchy; other Europeans, following philosophers Thomas Hobbes and Jean-Jacques Rousseau, contrasted "culture" with "the state of nature". According to Hobbes and Rousseau, the Native Americans who were being conquered by Europeans from the 16th centuries on were living in a state of nature; this opposition was expressed through the contrast between "civilized" and "uncivilized." According to this way of thinking, one could classify some countries and nations as more civilized than others and some people as more cultured than others. This contrast led to Herbert Spencer's theory of Social Darwinism and Lewis Henry Morgan's theory of cultural evolution. Just as some critics have argued that the distinction between high and low cultures is really an expression of the conflict between European elites and non-elites, some critics have argued that the distinction between civilized and uncivilized people is really an expression of the conflict between European colonial powers and their colonial subjects. +Other 19th-century critics, following Rousseau have accepted this differentiation between higher and lower culture, but have seen the refinement and sophistication of high culture as corrupting and unnatural developments that obscure and distort people's essential nature. These critics considered folk music (as produced by "the folk", i.e., rural, illiterate, peasants) to honestly express a natural way of life, while classical music seemed superficial and decadent. Equally, this view often portrayed indigenous peoples as "noble savages" living authentic and unblemished lives, uncomplicated and uncorrupted by the highly stratified capitalist systems of the West. +Although anthropologists worldwide refer to Tylor's definition of culture, in the 20th century "culture" emerged as the central and unifying concept of American anthropology, where it most commonly refers to the universal human capacity to classify and encode human experiences symbolically, and to communicate symbolically encoded experiences socially.[citation needed] American anthropology is organized into four fields, each of which plays an important role in research on culture: biological anthropology, linguistic anthropology, cultural anthropology, and archaeology. +The sociology of culture concerns culture—usually understood as the ensemble of symbolic codes used by a society—as manifested in society. For Georg Simmel (1858–1918), culture referred to "the cultivation of individuals through the agency of external forms which have been objectified in the course of history". Culture in the sociological field can be defined as the ways of thinking, the ways of acting, and the material objects that together shape a people's way of life. Culture can be any of two types, non-material culture or material culture. Non-material culture refers to the non physical ideas that individuals have about their culture, including values, belief system, rules, norms, morals, language, organizations, and institutions. While Material culture is the physical evidence of a culture in the objects and architecture they make, or have made. The term tends to be relevant only in archeological and anthropological studies, but it specifically means all material evidence which can be attributed to culture past or present. +Cultural sociology first emerged in Weimar Germany (1918–1933), where sociologists such as Alfred Weber used the term Kultursoziologie (cultural sociology). Cultural sociology was then "reinvented" in the English-speaking world as a product of the "cultural turn" of the 1960s, which ushered in structuralist and postmodern approaches to social science. This type of cultural sociology may loosely be regarded as an approach incorporating cultural analysis and critical theory. Cultural sociologists tend to reject scientific methods,[citation needed] instead hermeneutically focusing on words, artifacts and symbols. "Culture" has since become an important concept across many branches of sociology, including resolutely scientific fields like social stratification and social network analysis. As a result, there has been a recent influx of quantitative sociologists to the field. Thus there is now a growing group of sociologists of culture who are, confusingly, not cultural sociologists. These scholars reject the abstracted postmodern aspects of cultural sociology, and instead look for a theoretical backing in the more scientific vein of social psychology and cognitive science. "Cultural sociology" is one of the largest sections of the American Sociological Association. The British establishment of cultural studies means the latter is often taught as a loosely distinct discipline in the UK. +The sociology of culture grew from the intersection between sociology (as shaped by early theorists like Marx, Durkheim, and Weber) with the growing discipline of anthropology, where in researchers pioneered ethnographic strategies for describing and analyzing a variety of cultures around the world. Part of the legacy of the early development of the field lingers in the methods (much of cultural sociological research is qualitative), in the theories (a variety of critical approaches to sociology are central to current research communities), and in the substantive focus of the field. For instance, relationships between popular culture, political control, and social class were early and lasting concerns in the field. +In the United Kingdom, sociologists and other scholars influenced by Marxism, such as Stuart Hall (1932–2014) and Raymond Williams (1921–1988), developed cultural studies. Following nineteenth-century Romantics, they identified "culture" with consumption goods and leisure activities (such as art, music, film, food, sports, and clothing). Nevertheless, they saw patterns of consumption and leisure as determined by relations of production, which led them to focus on class relations and the organization of production. +In the United States, "Cultural Studies" focuses largely on the study of popular culture, that is, on the social meanings of mass-produced consumer and leisure goods. Richard Hoggart coined the term in 1964 when he founded the Birmingham Centre for Contemporary Cultural Studies or CCCS. It has since become strongly associated with Stuart Hall, who succeeded Hoggart as Director. Cultural studies in this sense, then, can be viewed as a limited concentration scoped on the intricacies of consumerism, which belongs to a wider culture sometimes referred to as "Western Civilization" or as "Globalism." +From the 1970s onward, Stuart Hall's pioneering work, along with that of his colleagues Paul Willis, Dick Hebdige, Tony Jefferson, and Angela McRobbie, created an international intellectual movement. As the field developed it began to combine political economy, communication, sociology, social theory, literary theory, media theory, film/video studies, cultural anthropology, philosophy, museum studies and art history to study cultural phenomena or cultural texts. In this field researchers often concentrate on how particular phenomena relate to matters of ideology, nationality, ethnicity, social class, and/or gender.[citation needed] Cultural studies has a concern with the meaning and practices of everyday life. These practices comprise the ways people do particular things (such as watching television, or eating out) in a given culture. This field studies the meanings and uses people attribute to various objects and practices. Specifically, culture involves those meanings and practices held independently of reason. Watching television in order to view a public perspective on a historical event should not be thought of as culture, unless referring to the medium of television itself, which may have been selected culturally; however, schoolchildren watching television after school with their friends in order to "fit in" certainly qualifies, since there is no grounded reason for one's participation in this practice. Recently, as capitalism has spread throughout the world (a process called globalization), cultural studies has begun[when?] to analyze local and global forms of resistance to Western hegemony.[citation needed] Globalization in this context can be defined as western civilization in other ways, it undermines the cultural integrity of other culture and it is therefore repressive, exploitative and harmful to most people in different places. +In the context of cultural studies, the idea of a text includes not only written language, but also films, photographs, fashion or hairstyles: the texts of cultural studies comprise all the meaningful artifacts of culture.[citation needed] Similarly, the discipline widens the concept of "culture". "Culture" for a cultural-studies researcher not only includes traditional high culture (the culture of ruling social groups) and popular culture, but also everyday meanings and practices. The last two, in fact, have become the main focus of cultural studies. A further and recent approach is comparative cultural studies, based on the disciplines of comparative literature and cultural studies.[citation needed] +Scholars in the United Kingdom and the United States developed somewhat different versions of cultural studies after the late 1970s. The British version of cultural studies had originated in the 1950s and 1960s, mainly under the influence first of Richard Hoggart, E. P. Thompson, and Raymond Williams, and later that of Stuart Hall and others at the Centre for Contemporary Cultural Studies at the University of Birmingham. This included overtly political, left-wing views, and criticisms of popular culture as "capitalist" mass culture; it absorbed some of the ideas of the Frankfurt School critique of the "culture industry" (i.e. mass culture). This emerges in the writings of early British cultural-studies scholars and their influences: see the work of (for example) Raymond Williams, Stuart Hall, Paul Willis, and Paul Gilroy. +In the United States, Lindlof and Taylor write, "Cultural studies [were] grounded in a pragmatic, liberal-pluralist tradition". The American version of cultural studies initially concerned itself more with understanding the subjective and appropriative side of audience reactions to, and uses of, mass culture; for example, American cultural-studies advocates wrote about the liberatory aspects of fandom.[citation needed] The distinction between American and British strands, however, has faded.[citation needed] Some researchers, especially in early British cultural studies, apply a Marxist model to the field. This strain of thinking has some influence from the Frankfurt School, but especially from the structuralist Marxism of Louis Althusser and others. The main focus of an orthodox Marxist approach concentrates on the production of meaning. This model assumes a mass production of culture and identifies power as residing with those producing cultural artifacts. In a Marxist view, those who control the means of production (the economic base) essentially control a culture.[citation needed] Other approaches to cultural studies, such as feminist cultural studies and later American developments of the field, distance themselves from this view. They criticize the Marxist assumption of a single, dominant meaning, shared by all, for any cultural product. The non-Marxist approaches suggest that different ways of consuming cultural artifacts affect the meaning of the product. This view comes through in the book Doing Cultural Studies: The Story of the Sony Walkman (by Paul du Gay et al.), which seeks to challenge the notion that those who produce commodities control the meanings that people attribute to them. Feminist cultural analyst, theorist and art historian Griselda Pollock contributed to cultural studies from viewpoints of art history and psychoanalysis. The writer Julia Kristeva is among influential voices at the turn of the century, contributing to cultural studies from the field of art and psychoanalytical French feminism.[citation needed] +Raimon Panikkar pointed out 29 ways in which cultural change can be brought about. Some of these are: growth, development, evolution, involution, renovation, reconception, reform, innovation, revivalism, revolution, mutation, progress, diffusion, osmosis, borrowing, eclecticism, syncretism, modernization, indigenization, and transformation. Hence Modernization could be similar or related to the enlightenment but a 'looser' term set to ideal and values that flourish. a belief in objectivity progress. Also seen as a belief in a secular society (free from religious influences) example objective and rational, science vs religion and finally been modern means not being religious. diff --git a/e2e/samples/test_documents/Edmund_Burke.txt b/e2e/samples/test_documents/Edmund_Burke.txt new file mode 100644 index 0000000000..47ff51559d --- /dev/null +++ b/e2e/samples/test_documents/Edmund_Burke.txt @@ -0,0 +1,40 @@ +Burke was born in Dublin, Ireland. His mother Mary née Nagle (c. 1702 – 1770) was a Roman Catholic who hailed from a déclassé County Cork family (and a cousin of Nano Nagle), whereas his father, a successful solicitor, Richard (died 1761), was a member of the Church of Ireland; it remains unclear whether this is the same Richard Burke who converted from Catholicism. The Burke dynasty descends from an Anglo-Norman knight surnamed de Burgh (latinised as de Burgo) who arrived in Ireland in 1185 following Henry II of England's 1171 invasion of Ireland. +In 1744, Burke started at Trinity College Dublin, a Protestant establishment, which up until 1793, did not permit Catholics to take degrees. In 1747, he set up a debating society, "Edmund Burke's Club", which, in 1770, merged with TCD's Historical Club to form the College Historical Society; it is the oldest undergraduate society in the world. The minutes of the meetings of Burke's Club remain in the collection of the Historical Society. Burke graduated from Trinity in 1748. Burke's father wanted him to read Law, and with this in mind he went to London in 1750, where he entered the Middle Temple, before soon giving up legal study to travel in Continental Europe. After eschewing the Law, he pursued a livelihood through writing. +Burke claimed that Bolingbroke's arguments against revealed religion could apply to all social and civil institutions as well. Lord Chesterfield and Bishop Warburton (and others) initially thought that the work was genuinely by Bolingbroke rather than a satire. All the reviews of the work were positive, with critics especially appreciative of Burke's quality of writing. Some reviewers failed to notice the ironic nature of the book, which led to Burke stating in the preface to the second edition (1757) that it was a satire. +Richard Hurd believed that Burke's imitation was near-perfect and that this defeated his purpose: an ironist "should take care by a constant exaggeration to make the ridicule shine through the Imitation. Whereas this Vindication is everywhere enforc'd, not only in the language, and on the principles of L. Bol., but with so apparent, or rather so real an earnestness, that half his purpose is sacrificed to the other". A minority of scholars have taken the position that, in fact, Burke did write the Vindication in earnest, later disowning it only for political reasons. +On 25 February 1757, Burke signed a contract with Robert Dodsley to write a "history of England from the time of Julius Caesar to the end of the reign of Queen Anne", its length being eighty quarto sheets (640 pages), nearly 400,000 words. It was to be submitted for publication by Christmas 1758. Burke completed the work to the year 1216 and stopped; it was not published until after Burke's death, being included in an 1812 collection of his works, entitled An Essay Towards an Abridgement of the English History. G. M. Young did not value Burke's history and claimed that it was "demonstrably a translation from the French". Lord Acton, on commenting on the story that Burke stopped his history because David Hume published his, said "it is ever to be regretted that the reverse did not occur". +During the year following that contract, with Dodsley, Burke founded the influential Annual Register, a publication in which various authors evaluated the international political events of the previous year. The extent to which Burke contributed to the Annual Register is unclear: in his biography of Burke, Robert Murray quotes the Register as evidence of Burke's opinions, yet Philip Magnus in his biography does not cite it directly as a reference. Burke remained the chief editor of the publication until at least 1789 and there is no evidence that any other writer contributed to it before 1766. +At about this same time, Burke was introduced to William Gerard Hamilton (known as "Single-speech Hamilton"). When Hamilton was appointed Chief Secretary for Ireland, Burke accompanied him to Dublin as his private secretary, a position he held for three years. In 1765 Burke became private secretary to the liberal Whig statesman, Charles, Marquess of Rockingham, then Prime Minister of Great Britain, who remained Burke's close friend and associate until his untimely death in 1782. Rockingham also introduced Burke as a Freemason. +Burke took a leading role in the debate regarding the constitutional limits to the executive authority of the king. He argued strongly against unrestrained royal power and for the role of political parties in maintaining a principled opposition capable of preventing abuses, either by the monarch, or by specific factions within the government. His most important publication in this regard was his Thoughts on the Cause of the Present Discontents of 23 April 1770. Burke identified the "discontents" as stemming from the "secret influence" of a neo-Tory group he labelled as, the "king's friends", whose system "comprehending the exterior and interior administrations, is commonly called, in the technical language of the Court, Double Cabinet". Britain needed a party with "an unshaken adherence to principle, and attachment to connexion, against every allurement of interest". Party divisions "whether operating for good or evil, are things inseparable from free government". +In May 1778, Burke supported a parliamentary motion revising restrictions on Irish trade. His constituents, citizens of the great trading city of Bristol, however urged Burke to oppose free trade with Ireland. Burke resisted their protestations and said: "If, from this conduct, I shall forfeit their suffrages at an ensuing election, it will stand on record an example to future representatives of the Commons of England, that one man at least had dared to resist the desires of his constituents when his judgment assured him they were wrong". +Burke was not merely presenting a peace agreement to Parliament; rather, he stepped forward with four reasons against using force, carefully reasoned. He laid out his objections in an orderly manner, focusing on one before moving to the next. His first concern was that the use of force would have to be temporary, and that the uprisings and objections to British governance in America would not be. Second, Burke worried about the uncertainty surrounding whether Britain would win a conflict in America. "An armament", Burke said, "is not a victory". Third, Burke brought up the issue of impairment; it would do the British Government no good to engage in a scorched earth war and have the object they desired (America) become damaged or even useless. The American colonists could always retreat into the mountains, but the land they left behind would most likely be unusable, whether by accident or design. The fourth and final reason to avoid the use of force was experience; the British had never attempted to rein in an unruly colony by force, and they did not know if it could be done, let alone accomplished thousands of miles away from home. Not only were all of these concerns reasonable, but some turned out to be prophetic – the American colonists did not surrender, even when things looked extremely bleak, and the British were ultimately unsuccessful in their attempts to win a war fought on American soil. +Among the reasons this speech was so greatly admired was its passage on Lord Bathurst (1684–1775); Burke describes an angel in 1704 prophesying to Bathurst the future greatness of England and also of America: "Young man, There is America – which at this day serves little more than to amuse you with stories of savage men, and uncouth manners; yet shall, before you taste of death, shew itself equal to the whole of that commerce which now attracts the envy of the world". Samuel Johnson was so irritated at hearing it continually praised, that he made a parody of it, where the devil appears to a young Whig and predicts that in short time, Whiggism will poison even the paradise of America! +The administration of Lord North (1770–1782) tried to defeat the colonist rebellion by military force. British and American forces clashed in 1775 and, in 1776, came the American Declaration of Independence. Burke was appalled by celebrations in Britain of the defeat of the Americans at New York and Pennsylvania. He claimed the English national character was being changed by this authoritarianism. Burke wrote: "As to the good people of England, they seem to partake every day more and more of the Character of that administration which they have been induced to tolerate. I am satisfied, that within a few years there has been a great Change in the National Character. We seem no longer that eager, inquisitive, jealous, fiery people, which we have been formerly". +The Paymaster General Act 1782 ended the post as a lucrative sinecure. Previously, Paymasters had been able to draw on money from HM Treasury at their discretion. Now they were required to put the money they had requested to withdraw from the Treasury into the Bank of England, from where it was to be withdrawn for specific purposes. The Treasury would receive monthly statements of the Paymaster's balance at the Bank. This act was repealed by Shelburne's administration, but the act that replaced it repeated verbatim almost the whole text of the Burke Act. +Burke was a leading sceptic with respect to democracy. While admitting that theoretically, in some cases it might be desirable, he insisted a democratic government in Britain in his day would not only be inept, but also oppressive. He opposed democracy for three basic reasons. First, government required a degree of intelligence and breadth of knowledge of the sort that occurred rarely among the common people. Second, he thought that if they had the vote, common people had dangerous and angry passions that could be aroused easily by demagogues; he feared that the authoritarian impulses that could be empowered by these passions would undermine cherished traditions and established religion, leading to violence and confiscation of property. Third, Burke warned that democracy would create a tyranny over unpopular minorities, who needed the protection of the upper classes. +For years Burke pursued impeachment efforts against Warren Hastings, formerly Governor-General of Bengal, that resulted in the trial during 1786. His interaction with the British dominion of India began well before Hastings' impeachment trial. For two decades prior to the impeachment, Parliament had dealt with the Indian issue. This trial was the pinnacle of years of unrest and deliberation. In 1781 Burke was first able to delve into the issues surrounding the East India Company when he was appointed Chairman of the Commons Select Committee on East Indian Affairs—from that point until the end of the trial; India was Burke's primary concern. This committee was charged "to investigate alleged injustices in Bengal, the war with Hyder Ali, and other Indian difficulties". While Burke and the committee focused their attention on these matters, a second 'secret' committee was formed to assess the same issues. Both committee reports were written by Burke. Among other purposes, the reports conveyed to the Indian princes that Britain would not wage war on them, along with demanding that the HEIC recall Hastings. This was Burke's first call for substantive change regarding imperial practices. When addressing the whole House of Commons regarding the committee report, Burke described the Indian issue as one that "began 'in commerce' but 'ended in empire.'" +On 4 April 1786, Burke presented the Commons with the Article of Charge of High Crimes and Misdemeanors against Hastings. The impeachment in Westminster Hall, which did not begin until 14 February 1788, would be the "first major public discursive event of its kind in England", bringing the morality and duty of imperialism to the forefront of public perception. Burke already was known for his eloquent rhetorical skills and his involvement in the trial only enhanced its popularity and significance. Burke's indictment, fuelled by emotional indignation, branded Hastings a 'captain-general of iniquity'; who never dined without 'creating a famine'; whose heart was 'gangrened to the core', and who resembled both a 'spider of Hell' and a 'ravenous vulture devouring the carcasses of the dead'. The House of Commons eventually impeached Hastings, but subsequently, the House of Lords acquitted him of all charges. +Initially, Burke did not condemn the French Revolution. In a letter of 9 August 1789, Burke wrote: "England gazing with astonishment at a French struggle for Liberty and not knowing whether to blame or to applaud! The thing indeed, though I thought I saw something like it in progress for several years, has still something in it paradoxical and Mysterious. The spirit it is impossible not to admire; but the old Parisian ferocity has broken out in a shocking manner". The events of 5–6 October 1789, when a crowd of Parisian women marched on Versailles to compel King Louis XVI to return to Paris, turned Burke against it. In a letter to his son, Richard Burke, dated 10 October he said: "This day I heard from Laurence who has sent me papers confirming the portentous state of France—where the Elements which compose Human Society seem all to be dissolved, and a world of Monsters to be produced in the place of it—where Mirabeau presides as the Grand Anarch; and the late Grand Monarch makes a figure as ridiculous as pitiable". On 4 November Charles-Jean-François Depont wrote to Burke, requesting that he endorse the Revolution. Burke replied that any critical language of it by him should be taken "as no more than the expression of doubt" but he added: "You may have subverted Monarchy, but not recover'd freedom". In the same month he described France as "a country undone". Burke's first public condemnation of the Revolution occurred on the debate in Parliament on the army estimates on 9 February 1790, provoked by praise of the Revolution by Pitt and Fox: +In January 1790, Burke read Dr. Richard Price's sermon of 4 November 1789 entitled, A Discourse on the Love of our Country, to the Revolution Society. That society had been founded to commemorate the Glorious Revolution of 1688. In this sermon Price espoused the philosophy of universal "Rights of Men". Price argued that love of our country "does not imply any conviction of the superior value of it to other countries, or any particular preference of its laws and constitution of government". Instead, Price asserted that Englishmen should see themselves "more as citizens of the world than as members of any particular community". +Immediately after reading Price's sermon, Burke wrote a draft of what eventually became, Reflections on the Revolution in France. On 13 February 1790, a notice in the press said that shortly, Burke would publish a pamphlet on the Revolution and its British supporters, however he spent the year revising and expanding it. On 1 November he finally published the Reflections and it was an immediate best-seller. Priced at five shillings, it was more expensive than most political pamphlets, but by the end of 1790, it had gone through ten printings and sold approximately 17,500 copies. A French translation appeared on 29 November and on 30 November the translator, Pierre-Gaëton Dupont, wrote to Burke saying 2,500 copies had already been sold. The French translation ran to ten printings by June 1791. +Burke put forward that "We fear God, we look up with awe to kings; with affection to parliaments; with duty to magistrates; with reverence to priests; and with respect to nobility. Why? Because when such ideas are brought before our minds, it is natural to be so affected". Burke defended this prejudice on the grounds that it is "the general bank and capital of nations, and of ages" and superior to individual reason, which is small in comparison. "Prejudice", Burke claimed, "is of ready application in the emergency; it previously engages the mind in a steady course of wisdom and virtue, and does not leave the man hesitating in the moment of decision, skeptical, puzzled, and unresolved. Prejudice renders a man's virtue his habit". Burke criticised social contract theory by claiming that society is indeed, a contract, but "a partnership not only between those who are living, but between those who are living, those who are dead, and those who are to be born". +The most famous passage in Burke's Reflections was his description of the events of 5–6 October 1789 and the part of Marie-Antoinette in them. Burke's account differs little from modern historians who have used primary sources. His use of flowery language to describe it, however, provoked both praise and criticism. Philip Francis wrote to Burke saying that what he wrote of Marie-Antoinette was "pure foppery". Edward Gibbon, however, reacted differently: "I adore his chivalry". Burke was informed by an Englishman who had talked with the Duchesse de Biron, that when Marie-Antoinette was reading the passage, she burst into tears and took considerable time to finish reading it. Price had rejoiced that the French king had been "led in triumph" during the October Days, but to Burke this symbolised the opposing revolutionary sentiment of the Jacobins and the natural sentiments of those who shared his own view with horror—that the ungallant assault on Marie-Antoinette—was a cowardly attack on a defenceless woman. +Louis XVI translated the Reflections "from end to end" into French. Fellow Whig MPs Richard Sheridan and Charles James Fox, disagreed with Burke and split with him. Fox thought the Reflections to be "in very bad taste" and "favouring Tory principles". Other Whigs such as the Duke of Portland and Earl Fitzwilliam privately agreed with Burke, but did not wish for a public breach with their Whig colleagues. Burke wrote on 29 November 1790: "I have received from the Duke of Portland, Lord Fitzwilliam, the Duke of Devonshire, Lord John Cavendish, Montagu (Frederick Montagu MP), and a long et cetera of the old Stamina of the Whiggs a most full approbation of the principles of that work and a kind indulgence to the execution". The Duke of Portland said in 1791 that when anyone criticised the Reflections to him, he informed them that he had recommended the book to his sons as containing the true Whig creed. +Burke's Reflections sparked a pamphlet war. Thomas Paine penned the Rights of Man in 1791 as a response to Burke; Mary Wollstonecraft published A Vindication of the Rights of Men and James Mackintosh wrote Vindiciae Gallicae. Mackintosh was the first to see the Reflections as "the manifesto of a Counter Revolution". Mackintosh later agreed with Burke's views, remarking in December 1796 after meeting him, that Burke was "minutely and accurately informed, to a wonderful exactness, with respect to every fact relating to the French Revolution". Mackintosh later said: "Burke was one of the first thinkers as well as one of the greatest orators of his time. He is without parallel in any age, excepting perhaps Lord Bacon and Cicero; and his works contain an ampler store of political and moral wisdom than can be found in any other writer whatever". +In November 1790, François-Louis-Thibault de Menonville, a member of the National Assembly of France, wrote to Burke, praising Reflections and requesting more "very refreshing mental food" that he could publish. This Burke did in April 1791 when he published A Letter to a Member of the National Assembly. Burke called for external forces to reverse the revolution and included an attack on the late French philosopher Jean-Jacques Rousseau, as being the subject of a personality cult that had developed in revolutionary France. Although Burke conceded that Rousseau sometimes showed "a considerable insight into human nature" he mostly was critical. Although he did not meet Rousseau on his visit to Britain in 1766–7 Burke was a friend of David Hume, with whom Rousseau had stayed. Burke said Rousseau "entertained no principle either to influence of his heart, or to guide his understanding—but vanity"—which he "was possessed to a degree little short of madness". He also cited Rousseau's Confessions as evidence that Rousseau had a life of "obscure and vulgar vices" that was not "chequered, or spotted here and there, with virtues, or even distinguished by a single good action". Burke contrasted Rousseau's theory of universal benevolence and his having sent his children to a foundling hospital: "a lover of his kind, but a hater of his kindred". +These events and the disagreements that arose from them within the Whig Party, led to its break-up and to the rupture of Burke's friendship with Fox. In debate in Parliament on Britain's relations with Russia, Fox praised the principles of the revolution, although Burke was not able to reply at this time as he was "overpowered by continued cries of question from his own side of the House". When Parliament was debating the Quebec Bill for a constitution for Canada, Fox praised the revolution and criticised some of Burke's arguments, such as hereditary power. On 6 May 1791, during another debate in Parliament on the Quebec Bill, Burke used the opportunity to answer Fox, and to condemn the new French Constitution and "the horrible consequences flowing from the French idea of the Rights of Man". Burke asserted that those ideas were the antithesis of both the British and the American constitutions. Burke was interrupted, and Fox intervened, saying that Burke should be allowed to carry on with his speech. A vote of censure was moved against Burke, however, for noticing the affairs of France, which was moved by Lord Sheffield and seconded by Fox. Pitt made a speech praising Burke, and Fox made a speech—both rebuking and complimenting Burke. He questioned the sincerity of Burke, who seemed to have forgotten the lessons he had learned from him, quoting from Burke's own speeches of fourteen and fifteen years before. +At this point, Fox whispered that there was "no loss of friendship". "I regret to say there is", Burke replied, "I have indeed made a great sacrifice; I have done my duty though I have lost my friend. There is something in the detested French constitution that envenoms every thing it touches". This provoked a reply from Fox, yet he was unable to give his speech for some time since he was overcome with tears and emotion, he appealed to Burke to remember their inalienable friendship, but also repeated his criticisms of Burke and uttered "unusually bitter sarcasms". This only aggravated the rupture between the two men. Burke demonstrated his separation from the party on 5 June 1791 by writing to Fitzwilliam, declining money from him. +Burke knew that many members of the Whig Party did not share Fox's views and he wanted to provoke them into condemning the French Revolution. Burke wrote that he wanted to represent the whole Whig party "as tolerating, and by a toleration, countenancing those proceedings" so that he could "stimulate them to a public declaration of what every one of their acquaintance privately knows to be...their sentiments". Therefore, on 3 August 1791 Burke published his Appeal from the New to the Old Whigs, in which he renewed his criticism of the radical revolutionary programmes inspired by the French Revolution and attacked the Whigs who supported them, as holding principles contrary to those traditionally held by the Whig party. +Although Whig grandees such as Portland and Fitzwilliam privately agreed with Burke's Appeal, they wished he had used more moderate language. Fitzwilliam saw the Appeal as containing "the doctrines I have sworn by, long and long since". Francis Basset, a backbench Whig MP, wrote to Burke: "...though for reasons which I will not now detail I did not then deliver my sentiments, I most perfectly differ from Mr. Fox & from the great Body of opposition on the French Revolution". Burke sent a copy of the Appeal to the king and the king requested a friend to communicate to Burke that he had read it "with great Satisfaction". Burke wrote of its reception: "Not one word from one of our party. They are secretly galled. They agree with me to a title; but they dare not speak out for fear of hurting Fox. ... They leave me to myself; they see that I can do myself justice". Charles Burney viewed it as "a most admirable book—the best & most useful on political subjects that I have ever seen" but believed the differences in the Whig Party between Burke and Fox should not be aired publicly. +Burke supported the war against revolutionary France, seeing Britain as fighting on the side of the royalists and émigres in a civil war, rather than fighting against the whole nation of France. Burke also supported the royalist uprising in La Vendée, describing it on 4 November 1793 in a letter to William Windham, as "the sole affair I have much heart in". Burke wrote to Henry Dundas on 7 October urging him to send reinforcements there, as he viewed it as the only theatre in the war that might lead to a march on Paris. Dundas did not follow Burke's advice, however. +Burke believed the Government was not taking the uprising seriously enough, a view reinforced by a letter he had received from the Prince Charles of France (S.A.R. le comte d'Artois), dated 23 October, requesting that he intercede on behalf of the royalists to the Government. Burke was forced to reply on 6 November: "I am not in His Majesty's Service; or at all consulted in his Affairs". Burke published his Remarks on the Policy of the Allies with Respect to France, begun in October, where he said: "I am sure every thing has shewn us that in this war with France, one Frenchman is worth twenty foreigners. La Vendée is a proof of this". +On 20 June 1794, Burke received a vote of thanks from the Commons for his services in the Hastings Trial and he immediately resigned his seat, being replaced by his son Richard. A tragic blow fell upon Burke with the loss of Richard in August 1794, to whom he was tenderly attached, and in whom he saw signs of promise, which were not patent to others and which, in fact, appear to have been non-existent (though this view may have rather reflected the fact that Richard Burke had worked successfully in the early battle for Catholic emancipation). King George III, whose favour he had gained by his attitude on the French Revolution, wished to create him Earl of Beaconsfield, but the death of his son deprived the opportunity of such an honour and all its attractions, so the only award he would accept was a pension of £2,500. Even this modest reward was attacked by the Duke of Bedford and the Earl of Lauderdale, to whom Burke replied in his Letter to a Noble Lord (1796): "It cannot at this time be too often repeated; line upon line; precept upon precept; until it comes into the currency of a proverb, To innovate is not to reform". He argued that he was rewarded on merit, but the Duke of Bedford received his rewards from inheritance alone, his ancestor being the original pensioner: "Mine was from a mild and benevolent sovereign; his from Henry the Eighth". Burke also hinted at what would happen to such people if their revolutionary ideas were implemented, and included a description of the British constitution: +Burke's last publications were the Letters on a Regicide Peace (October 1796), called forth by negotiations for peace with France by the Pitt government. Burke regarded this as appeasement, injurious to national dignity and honour. In his Second Letter, Burke wrote of the French Revolutionary Government: "Individuality is left out of their scheme of government. The State is all in all. Everything is referred to the production of force; afterwards, everything is trusted to the use of it. It is military in its principle, in its maxims, in its spirit, and in all its movements. The State has dominion and conquest for its sole objects—dominion over minds by proselytism, over bodies by arms". +This is held to be the first explanation of the modern concept of totalitarian state. Burke regarded the war with France as ideological, against an "armed doctrine". He wished that France would not be partitioned due to the effect this would have on the balance of power in Europe, and that the war was not against France, but against the revolutionaries governing her. Burke said: "It is not France extending a foreign empire over other nations: it is a sect aiming at universal empire, and beginning with the conquest of France". +In November 1795, there was a debate in Parliament on the high price of corn and Burke wrote a memorandum to Pitt on the subject. In December Samuel Whitbread MP introduced a bill giving magistrates the power to fix minimum wages and Fox said he would vote for it. This debate probably led Burke to editing his memorandum, as there appeared a notice that Burke would soon publish a letter on the subject to the Secretary of the Board of Agriculture, Arthur Young; but he failed to complete it. These fragments were inserted into the memorandum after his death and published posthumously in 1800 as, Thoughts and Details on Scarcity. In it, Burke expounded "some of the doctrines of political economists bearing upon agriculture as a trade". Burke criticised policies such as maximum prices and state regulation of wages, and set out what the limits of government should be: +Writing to a friend in May 1795, Burke surveyed the causes of discontent: "I think I can hardly overrate the malignity of the principles of Protestant ascendency, as they affect Ireland; or of Indianism [i.e. corporate tyranny, as practiced by the British East Indies Company], as they affect these countries, and as they affect Asia; or of Jacobinism, as they affect all Europe, and the state of human society itself. The last is the greatest evil". By March 1796, however Burke had changed his mind: "Our Government and our Laws are beset by two different Enemies, which are sapping its foundations, Indianism, and Jacobinism. In some Cases they act separately, in some they act in conjunction: But of this I am sure; that the first is the worst by far, and the hardest to deal with; and for this amongst other reasons, that it weakens discredits, and ruins that force, which ought to be employed with the greatest Credit and Energy against the other; and that it furnishes Jacobinism with its strongest arms against all formal Government". +Burke believed that property was essential to human life. Because of his conviction that people desire to be ruled and controlled, the division of property formed the basis for social structure, helping develop control within a property-based hierarchy. He viewed the social changes brought on by property as the natural order of events, which should be taking place as the human race progressed. With the division of property and the class system, he also believed that it kept the monarch in check to the needs of the classes beneath the monarch. Since property largely aligned or defined divisions of social class, class too, was seen as natural—part of a social agreement that the setting of persons into different classes, is the mutual benefit of all subjects. Concern for property is not Burke's only influence. As Christopher Hitchens summarises, "If modern conservatism can be held to derive from Burke, it is not just because he appealed to property owners in behalf of stability but also because he appealed to an everyday interest in the preservation of the ancestral and the immemorial." +In the nineteenth century Burke was praised by both liberals and conservatives. Burke's friend Philip Francis wrote that Burke "was a man who truly & prophetically foresaw all the consequences which would rise from the adoption of the French principles" but because Burke wrote with so much passion, people were doubtful of his arguments. William Windham spoke from the same bench in the House of Commons as Burke had, when he had separated from Fox, and an observer said Windham spoke "like the ghost of Burke" when he made a speech against peace with France in 1801. William Hazlitt, a political opponent of Burke, regarded him as amongst his three favourite writers (the others being Junius and Rousseau), and made it "a test of the sense and candour of any one belonging to the opposite party, whether he allowed Burke to be a great man". William Wordsworth was originally a supporter of the French Revolution and attacked Burke in 'A Letter to the Bishop of Llandaff' (1793), but by the early nineteenth century he had changed his mind and came to admire Burke. In his Two Addresses to the Freeholders of Westmorland Wordsworth called Burke "the most sagacious Politician of his age" whose predictions "time has verified". He later revised his poem The Prelude to include praise of Burke ("Genius of Burke! forgive the pen seduced/By specious wonders") and portrayed him as an old oak. Samuel Taylor Coleridge came to have a similar conversion: he had criticised Burke in The Watchman, but in his Friend (1809–10) Coleridge defended Burke from charges of inconsistency. Later, in his Biographia Literaria (1817) Coleridge hails Burke as a prophet and praises Burke for referring "habitually to principles. He was a scientific statesman; and therefore a seer". Henry Brougham wrote of Burke: "... all his predictions, save one momentary expression, had been more than fulfilled: anarchy and bloodshed had borne sway in France; conquest and convulsion had desolated Europe...the providence of mortals is not often able to penetrate so far as this into futurity". George Canning believed that Burke's Reflections "has been justified by the course of subsequent events; and almost every prophecy has been strictly fulfilled". In 1823 Canning wrote that he took Burke's "last works and words [as] the manual of my politics". The Conservative Prime Minister Benjamin Disraeli "was deeply penetrated with the spirit and sentiment of Burke's later writings". +The 19th-century Liberal Prime Minister William Ewart Gladstone considered Burke "a magazine of wisdom on Ireland and America" and in his diary recorded: "Made many extracts from Burke—sometimes almost divine". The Radical MP and anti-Corn Law activist Richard Cobden often praised Burke's Thoughts and Details on Scarcity. The Liberal historian Lord Acton considered Burke one of the three greatest Liberals, along with William Gladstone and Thomas Babington Macaulay. Lord Macaulay recorded in his diary: "I have now finished reading again most of Burke's works. Admirable! The greatest man since Milton". The Gladstonian Liberal MP John Morley published two books on Burke (including a biography) and was influenced by Burke, including his views on prejudice. The Cobdenite Radical Francis Hirst thought Burke deserved "a place among English libertarians, even though of all lovers of liberty and of all reformers he was the most conservative, the least abstract, always anxious to preserve and renovate rather than to innovate. In politics he resembled the modern architect who would restore an old house instead of pulling it down to construct a new one on the site". Burke's Reflections on the Revolution in France was controversial at the time of its publication, but after his death, it was to become his best known and most influential work, and a manifesto for Conservative thinking. +The historian Piers Brendon asserts that Burke laid the moral foundations for the British Empire, epitomised in the trial of Warren Hastings, that was ultimately to be its undoing: when Burke stated that "The British Empire must be governed on a plan of freedom, for it will be governed by no other", this was "...an ideological bacillus that would prove fatal. This was Edmund Burke's paternalistic doctrine that colonial government was a trust. It was to be so exercised for the benefit of subject people that they would eventually attain their birthright—freedom". As a consequence of this opinion, Burke objected to the opium trade, which he called a "smuggling adventure" and condemned "the great Disgrace of the British character in India". +Burke's religious writing comprises published works and commentary on the subject of religion. Burke's religious thought was grounded in the belief that religion is the foundation of civil society. He sharply criticised deism and atheism, and emphasised Christianity as a vehicle of social progress. Born in Ireland to a Catholic mother and a Protestant father, Burke vigorously defended the Anglican Church, but also demonstrated sensitivity to Catholic concerns. He linked the conservation of a state (established) religion with the preservation of citizens' constitutional liberties and highlighted Christianity's benefit not only to the believer's soul, but also to political arrangements. diff --git a/e2e/samples/test_documents/Freemasonry.txt b/e2e/samples/test_documents/Freemasonry.txt new file mode 100644 index 0000000000..e1671b3648 --- /dev/null +++ b/e2e/samples/test_documents/Freemasonry.txt @@ -0,0 +1,37 @@ +Relations between Grand Lodges are determined by the concept of Recognition. Each Grand Lodge maintains a list of other Grand Lodges that it recognises. When two Grand Lodges recognise and are in Masonic communication with each other, they are said to be in amity, and the brethren of each may visit each other's Lodges and interact Masonically. When two Grand Lodges are not in amity, inter-visitation is not allowed. There are many reasons why one Grand Lodge will withhold or withdraw recognition from another, but the two most common are Exclusive Jurisdiction and Regularity. +Since the middle of the 19th century, Masonic historians have sought the origins of the movement in a series of similar documents known as the Old Charges, dating from the Regius Poem in about 1425 to the beginning of the 18th century. Alluding to the membership of a lodge of operative masons, they relate a mythologised history of the craft, the duties of its grades, and the manner in which oaths of fidelity are to be taken on joining. The fifteenth century also sees the first evidence of ceremonial regalia. +A dispute during the Lausanne Congress of Supreme Councils of 1875 prompted the Grand Orient de France to commission a report by a Protestant pastor which concluded that, as Freemasonry was not a religion, it should not require a religious belief. The new constitutions read, "Its principles are absolute liberty of conscience and human solidarity", the existence of God and the immortality of the soul being struck out. It is possible that the immediate objections of the United Grand Lodge of England were at least partly motivated by the political tension between France and Britain at the time. The result was the withdrawal of recognition of the Grand Orient of France by the United Grand Lodge of England, a situation that continues today. +At the dawn of the Grand Lodge era, during the 1720s, James Anderson composed the first printed constitutions for Freemasons, the basis for most subsequent constitutions, which specifically excluded women from Freemasonry. As Freemasonry spread, continental masons began to include their ladies in Lodges of Adoption, which worked three degrees with the same names as the men's but different content. The French officially abandoned the experiment in the early 19th century. Later organisations with a similar aim emerged in the United States, but distinguished the names of the degrees from those of male masonry. +In contrast to Catholic allegations of rationalism and naturalism, Protestant objections are more likely to be based on allegations of mysticism, occultism, and even Satanism. Masonic scholar Albert Pike is often quoted (in some cases misquoted) by Protestant anti-Masons as an authority for the position of Masonry on these issues. However, Pike, although undoubtedly learned, was not a spokesman for Freemasonry and was also controversial among Freemasons in general. His writings represented his personal opinion only, and furthermore an opinion grounded in the attitudes and understandings of late 19th century Southern Freemasonry of the USA. Notably, his book carries in the preface a form of disclaimer from his own Grand Lodge. No one voice has ever spoken for the whole of Freemasonry. +In 1799, English Freemasonry almost came to a halt due to Parliamentary proclamation. In the wake of the French Revolution, the Unlawful Societies Act 1799 banned any meetings of groups that required their members to take an oath or obligation. The Grand Masters of both the Moderns and the Antients Grand Lodges called on Prime Minister William Pitt (who was not a Freemason) and explained to him that Freemasonry was a supporter of the law and lawfully constituted authority and was much involved in charitable work. As a result, Freemasonry was specifically exempted from the terms of the Act, provided that each private lodge's Secretary placed with the local "Clerk of the Peace" a list of the members of his lodge once a year. This continued until 1967 when the obligation of the provision was rescinded by Parliament. +In some countries anti-Masonry is often related to antisemitism and anti-Zionism. For example, In 1980, the Iraqi legal and penal code was changed by Saddam Hussein's ruling Ba'ath Party, making it a felony to "promote or acclaim Zionist principles, including Freemasonry, or who associate [themselves] with Zionist organisations". Professor Andrew Prescott of the University of Sheffield writes: "Since at least the time of the Protocols of the Elders of Zion, antisemitism has gone hand in hand with anti-masonry, so it is not surprising that allegations that 11 September was a Zionist plot have been accompanied by suggestions that the attacks were inspired by a masonic world order". +The bulk of Masonic ritual consists of degree ceremonies. Candidates for Freemasonry are progressively initiated into Freemasonry, first in the degree of Entered Apprentice. Some time later, in a separate ceremony, they will be passed to the degree of Fellowcraft, and finally they will be raised to the degree of Master Mason. In all of these ceremonies, the candidate is entrusted with passwords, signs and grips peculiar to his new rank. Another ceremony is the annual installation of the Master and officers of the Lodge. In some jurisdictions Installed Master is valued as a separate rank, with its own secrets to distinguish its members. In other jurisdictions, the grade is not recognised, and no inner ceremony conveys new secrets during the installation of a new Master of the Lodge. +English Freemasonry spread to France in the 1720s, first as lodges of expatriates and exiled Jacobites, and then as distinctively French lodges which still follow the ritual of the Moderns. From France and England, Freemasonry spread to most of Continental Europe during the course of the 18th century. The Grande Loge de France formed under the Grand Mastership of the Duke of Clermont, who exercised only nominal authority. His successor, the Duke of Orléans, reconstituted the central body as the Grand Orient de France in 1773. Briefly eclipsed during the French Revolution, French Freemasonry continued to grow in the next century. +The majority of Freemasonry considers the Liberal (Continental) strand to be Irregular, and thus withhold recognition. For the Continental lodges, however, having a different approach to Freemasonry was not a reason for severing masonic ties. In 1961, an umbrella organisation, Centre de Liaison et d'Information des Puissances maçonniques Signataires de l'Appel de Strasbourg (CLIPSAS) was set up, which today provides a forum for most of these Grand Lodges and Grand Orients worldwide. Included in the list of over 70 Grand Lodges and Grand Orients are representatives of all three of the above categories, including mixed and women's organisations. The United Grand Lodge of England does not communicate with any of these jurisdictions, and expects its allies to follow suit. This creates the distinction between Anglo-American and Continental Freemasonry. +The denomination with the longest history of objection to Freemasonry is the Roman Catholic Church. The objections raised by the Roman Catholic Church are based on the allegation that Masonry teaches a naturalistic deistic religion which is in conflict with Church doctrine. A number of Papal pronouncements have been issued against Freemasonry. The first was Pope Clement XII's In eminenti apostolatus, 28 April 1738; the most recent was Pope Leo XIII's Ab apostolici, 15 October 1890. The 1917 Code of Canon Law explicitly declared that joining Freemasonry entailed automatic excommunication, and banned books favouring Freemasonry. +In 1933, the Orthodox Church of Greece officially declared that being a Freemason constitutes an act of apostasy and thus, until he repents, the person involved with Freemasonry cannot partake of the Eucharist. This has been generally affirmed throughout the whole Eastern Orthodox Church. The Orthodox critique of Freemasonry agrees with both the Roman Catholic and Protestant versions: "Freemasonry cannot be at all compatible with Christianity as far as it is a secret organisation, acting and teaching in mystery and secret and deifying rationalism." +In addition, most Grand Lodges require the candidate to declare a belief in a Supreme Being. In a few cases, the candidate may be required to be of a specific religion. The form of Freemasonry most common in Scandinavia (known as the Swedish Rite), for example, accepts only Christians. At the other end of the spectrum, "Liberal" or Continental Freemasonry, exemplified by the Grand Orient de France, does not require a declaration of belief in any deity, and accepts atheists (a cause of discord with the rest of Freemasonry). +Exclusive Jurisdiction is a concept whereby only one Grand Lodge will be recognised in any geographical area. If two Grand Lodges claim jurisdiction over the same area, the other Grand Lodges will have to choose between them, and they may not all decide to recognise the same one. (In 1849, for example, the Grand Lodge of New York split into two rival factions, each claiming to be the legitimate Grand Lodge. Other Grand Lodges had to choose between them until the schism was healed.) Exclusive Jurisdiction can be waived when the two over-lapping Grand Lodges are themselves in Amity and agree to share jurisdiction (for example, since the Grand Lodge of Connecticut is in Amity with the Prince Hall Grand Lodge of Connecticut, the principle of Exclusive Jurisdiction does not apply, and other Grand Lodges may recognise both). +There is no clear mechanism by which these local trade organisations became today's Masonic Lodges, but the earliest rituals and passwords known, from operative lodges around the turn of the 17th–18th centuries, show continuity with the rituals developed in the later 18th century by accepted or speculative Masons, as those members who did not practice the physical craft came to be known. The minutes of the Lodge of Edinburgh (Mary's Chapel) No. 1 in Scotland show a continuity from an operative lodge in 1598 to a modern speculative Lodge. It is reputed to be the oldest Masonic Lodge in the world. +Prince Hall Freemasonry exists because of the refusal of early American lodges to admit African-Americans. In 1775, an African-American named Prince Hall, along with fourteen other African-Americans, was initiated into a British military lodge with a warrant from the Grand Lodge of Ireland, having failed to obtain admission from the other lodges in Boston. When the military Lodge left North America, those fifteen men were given the authority to meet as a Lodge, but not to initiate Masons. In 1784, these individuals obtained a Warrant from the Premier Grand Lodge of England (GLE) and formed African Lodge, Number 459. When the UGLE was formed in 1813, all U.S.-based Lodges were stricken from their rolls – due largely to the War of 1812. Thus, separated from both UGLE and any concordantly recognised U.S. Grand Lodge, African Lodge re-titled itself as the African Lodge, Number 1 – and became a de facto "Grand Lodge" (this Lodge is not to be confused with the various Grand Lodges on the Continent of Africa). As with the rest of U.S. Freemasonry, Prince Hall Freemasonry soon grew and organised on a Grand Lodge system for each state. +Maria Deraismes was initiated into Freemasonry in 1882, then resigned to allow her lodge to rejoin their Grand Lodge. Having failed to achieve acceptance from any masonic governing body, she and Georges Martin started a mixed masonic lodge that actually worked masonic ritual. Annie Besant spread the phenomenon to the English speaking world. Disagreements over ritual led to the formation of exclusively female bodies of Freemasons in England, which spread to other countries. Meanwhile, the French had re-invented Adoption as an all-female lodge in 1901, only to cast it aside again in 1935. The lodges, however, continued to meet, which gave rise, in 1959, to a body of women practising continental Freemasonry. +Many Islamic anti-Masonic arguments are closely tied to both antisemitism and Anti-Zionism, though other criticisms are made such as linking Freemasonry to al-Masih ad-Dajjal (the false Messiah). Some Muslim anti-Masons argue that Freemasonry promotes the interests of the Jews around the world and that one of its aims is to destroy the Al-Aqsa Mosque in order to rebuild the Temple of Solomon in Jerusalem. In article 28 of its Covenant, Hamas states that Freemasonry, Rotary, and other similar groups "work in the interest of Zionism and according to its instructions ..." +The preserved records of the Reichssicherheitshauptamt (the Reich Security Main Office) show the persecution of Freemasons during the Holocaust. RSHA Amt VII (Written Records) was overseen by Professor Franz Six and was responsible for "ideological" tasks, by which was meant the creation of antisemitic and anti-Masonic propaganda. While the number is not accurately known, it is estimated that between 80,000 and 200,000 Freemasons were killed under the Nazi regime. Masonic concentration camp inmates were graded as political prisoners and wore an inverted red triangle. +Freemasonry consists of fraternal organisations that trace their origins to the local fraternities of stonemasons, which from the end of the fourteenth century regulated the qualifications of stonemasons and their interaction with authorities and clients. The degrees of freemasonry retain the three grades of medieval craft guilds, those of Apprentice, Journeyman or fellow (now called Fellowcraft), and Master Mason. These are the degrees offered by Craft (or Blue Lodge) Freemasonry. Members of these organisations are known as Freemasons or Masons. There are additional degrees, which vary with locality and jurisdiction, and are usually administered by different bodies than the craft degrees. +Candidates for Freemasonry will have met most active members of the Lodge they are joining before they are initiated. The process varies between jurisdictions, but the candidate will typically have been introduced by a friend at a Lodge social function, or at some form of open evening in the Lodge. In modern times, interested people often track down a local Lodge through the Internet. The onus is on candidates to ask to join; while candidates may be encouraged to ask, they are never invited. Once the initial inquiry is made, an interview usually follows to determine the candidate's suitability. If the candidate decides to proceed from here, the Lodge ballots on the application before he (or she, depending on the Masonic Jurisdiction) can be accepted. +Freemasonry, as it exists in various forms all over the world, has a membership estimated by the United Grand Lodge of England at around six million worldwide. The fraternity is administratively organised into independent Grand Lodges (or sometimes Grand Orients), each of which governs its own Masonic jurisdiction, which consists of subordinate (or constituent) Lodges. The largest single jurisdiction, in terms of membership, is the United Grand Lodge of England (with a membership estimated at around a quarter million). The Grand Lodge of Scotland and Grand Lodge of Ireland (taken together) have approximately 150,000 members. In the United States total membership is just under two million. +The idea of Masonic brotherhood probably descends from a 16th-century legal definition of a brother as one who has taken an oath of mutual support to another. Accordingly, Masons swear at each degree to keep the contents of that degree secret, and to support and protect their brethren unless they have broken the law. In most Lodges the oath or obligation is taken on a Volume of Sacred Law, whichever book of divine revelation is appropriate to the religious beliefs of the individual brother (usually the Bible in the Anglo-American tradition). In Progressive continental Freemasonry, books other than scripture are permissible, a cause of rupture between Grand Lodges. +The earliest known American lodges were in Pennsylvania. The Collector for the port of Pennsylvania, John Moore, wrote of attending lodges there in 1715, two years before the formation of the first Grand Lodge in London. The Premier Grand Lodge of England appointed a Provincial Grand Master for North America in 1731, based in Pennsylvania. Other lodges in the colony obtained authorisations from the later Antient Grand Lodge of England, the Grand Lodge of Scotland, and the Grand Lodge of Ireland, which was particularly well represented in the travelling lodges of the British Army. Many lodges came into existence with no warrant from any Grand Lodge, applying and paying for their authorisation only after they were confident of their own survival. +Masonic lodges existed in Iraq as early as 1917, when the first lodge under the United Grand Lodge of England (UGLE) was opened. Nine lodges under UGLE existed by the 1950s, and a Scottish lodge was formed in 1923. However, the position changed following the revolution, and all lodges were forced to close in 1965. This position was later reinforced under Saddam Hussein; the death penalty was "prescribed" for those who "promote or acclaim Zionist principles, including freemasonry, or who associate [themselves] with Zionist organisations." +The ritual form on which the Grand Orient of France was based was abolished in England in the events leading to the formation of the United Grand Lodge of England in 1813. However the two jurisdictions continued in amity (mutual recognition) until events of the 1860s and 1870s drove a seemingly permanent wedge between them. In 1868 the Supreme Council of the Ancient and Accepted Scottish Rite of the State of Louisiana appeared in the jurisdiction of the Grand Lodge of Louisiana, recognised by the Grand Orient de France, but regarded by the older body as an invasion of their jurisdiction. The new Scottish rite body admitted blacks, and the resolution of the Grand Orient the following year that neither colour, race, nor religion could disqualify a man from Masonry prompted the Grand Lodge to withdraw recognition, and it persuaded other American Grand Lodges to do the same. +In 1983, the Church issued a new code of canon law. Unlike its predecessor, the 1983 Code of Canon Law did not explicitly name Masonic orders among the secret societies it condemns. It states: "A person who joins an association which plots against the Church is to be punished with a just penalty; one who promotes or takes office in such an association is to be punished with an interdict." This named omission of Masonic orders caused both Catholics and Freemasons to believe that the ban on Catholics becoming Freemasons may have been lifted, especially after the perceived liberalisation of Vatican II. However, the matter was clarified when Cardinal Joseph Ratzinger (later Pope Benedict XVI), as the Prefect of the Congregation for the Doctrine of the Faith, issued a Declaration on Masonic Associations, which states: "... the Church's negative judgment in regard to Masonic association remains unchanged since their principles have always been considered irreconcilable with the doctrine of the Church and therefore membership in them remains forbidden. The faithful who enroll in Masonic associations are in a state of grave sin and may not receive Holy Communion." For its part, Freemasonry has never objected to Catholics joining their fraternity. Those Grand Lodges in amity with UGLE deny the Church's claims. The UGLE now states that "Freemasonry does not seek to replace a Mason's religion or provide a substitute for it." +Even in modern democracies, Freemasonry is sometimes viewed with distrust. In the UK, Masons working in the justice system, such as judges and police officers, were from 1999 to 2009 required to disclose their membership. While a parliamentary inquiry found that there has been no evidence of wrongdoing, it was felt that any potential loyalties Masons might have, based on their vows to support fellow Masons, should be transparent to the public. The policy of requiring a declaration of masonic membership of applicants for judicial office (judges and magistrates) was ended in 2009 by Justice Secretary Jack Straw (who had initiated the requirement in the 1990s). Straw stated that the rule was considered disproportionate, since no impropriety or malpractice had been shown as a result of judges being Freemasons. +The Masonic Lodge is the basic organisational unit of Freemasonry. The Lodge meets regularly to conduct the usual formal business of any small organisation (pay bills, organise social and charitable events, elect new members, etc.). In addition to business, the meeting may perform a ceremony to confer a Masonic degree or receive a lecture, which is usually on some aspect of Masonic history or ritual. At the conclusion of the meeting, the Lodge might adjourn for a formal dinner, or festive board, sometimes involving toasting and song. +During the ceremony of initiation, the candidate is expected to swear (usually on a volume of sacred text appropriate to his personal religious faith) to fulfil certain obligations as a Mason. In the course of three degrees, new masons will promise to keep the secrets of their degree from lower degrees and outsiders, and to support a fellow Mason in distress (as far as practicality and the law permit). There is instruction as to the duties of a Freemason, but on the whole, Freemasons are left to explore the craft in the manner they find most satisfying. Some will further explore the ritual and symbolism of the craft, others will focus their involvement on the social side of the Lodge, while still others will concentrate on the charitable functions of the lodge. +Regularity is a concept based on adherence to Masonic Landmarks, the basic membership requirements, tenets and rituals of the craft. Each Grand Lodge sets its own definition of what these landmarks are, and thus what is Regular and what is Irregular (and the definitions do not necessarily agree between Grand Lodges). Essentially, every Grand Lodge will hold that its landmarks (its requirements, tenets and rituals) are Regular, and judge other Grand Lodges based on those. If the differences are significant, one Grand Lodge may declare the other "Irregular" and withdraw or withhold recognition. +All Freemasons begin their journey in the "craft" by being progressively initiated, passed and raised into the three degrees of Craft, or Blue Lodge Masonry. During these three rituals, the candidate is progressively taught the meanings of the Lodge symbols, and entrusted with grips, signs and words to signify to other Masons that he has been so initiated. The initiations are part allegory and part lecture, and revolve around the construction of the Temple of Solomon, and the artistry and death of his chief architect, Hiram Abiff. The degrees are those of Entered apprentice, Fellowcraft and Master Mason. While many different versions of these rituals exist, with at least two different lodge layouts and versions of the Hiram myth, each version is recognisable to any Freemason from any jurisdiction. +The first Grand Lodge, the Grand Lodge of London and Westminster (later called the Grand Lodge of England (GLE)), was founded on 24 June 1717, when four existing London Lodges met for a joint dinner. Many English Lodges joined the new regulatory body, which itself entered a period of self-publicity and expansion. However, many Lodges could not endorse changes which some Lodges of the GLE made to the ritual (they came to be known as the Moderns), and a few of these formed a rival Grand Lodge on 17 July 1751, which they called the "Antient Grand Lodge of England." These two Grand Lodges vied for supremacy until the Moderns promised to return to the ancient ritual. They united on 27 December 1813 to form the United Grand Lodge of England (UGLE). +Widespread segregation in 19th- and early 20th-century North America made it difficult for African-Americans to join Lodges outside of Prince Hall jurisdictions – and impossible for inter-jurisdiction recognition between the parallel U.S. Masonic authorities. By the 1980s, such discrimination was a thing of the past, and today most U.S. Grand Lodges recognise their Prince Hall counterparts, and the authorities of both traditions are working towards full recognition. The United Grand Lodge of England has no problem with recognising Prince Hall Grand Lodges. While celebrating their heritage as lodges of black Americans, Prince Hall is open to all men regardless of race or religion. +In general, Continental Freemasonry is sympathetic to Freemasonry amongst women, dating from the 1890s when French lodges assisted the emergent co-masonic movement by promoting enough of their members to the 33rd degree of the Ancient and Accepted Scottish Rite to allow them, in 1899, to form their own grand council, recognised by the other Continental Grand Councils of that Rite. The United Grand Lodge of England issued a statement in 1999 recognising the two women's grand lodges there to be regular in all but the participants. While they were not, therefore, recognised as regular, they were part of Freemasonry "in general". The attitude of most regular Anglo-American grand lodges remains that women Freemasons are not legitimate Masons. +Since the founding of Freemasonry, many Bishops of the Church of England have been Freemasons, such as Archbishop Geoffrey Fisher. In the past, few members of the Church of England would have seen any incongruity in concurrently adhering to Anglican Christianity and practicing Freemasonry. In recent decades, however, reservations about Freemasonry have increased within Anglicanism, perhaps due to the increasing prominence of the evangelical wing of the church. The former Archbishop of Canterbury, Dr Rowan Williams, appeared to harbour some reservations about Masonic ritual, whilst being anxious to avoid causing offence to Freemasons inside and outside the Church of England. In 2003 he felt it necessary to apologise to British Freemasons after he said that their beliefs were incompatible with Christianity and that he had barred the appointment of Freemasons to senior posts in his diocese when he was Bishop of Monmouth. +In Italy, Freemasonry has become linked to a scandal concerning the Propaganda Due lodge (a.k.a. P2). This lodge was chartered by the Grande Oriente d'Italia in 1877, as a lodge for visiting Masons unable to attend their own lodges. Under Licio Gelli's leadership, in the late 1970s, P2 became involved in the financial scandals that nearly bankrupted the Vatican Bank. However, by this time the lodge was operating independently and irregularly, as the Grand Orient had revoked its charter and expelled Gelli in 1976. diff --git a/e2e/samples/test_documents/John_Kerry.txt b/e2e/samples/test_documents/John_Kerry.txt new file mode 100644 index 0000000000..a9a70cc7ee --- /dev/null +++ b/e2e/samples/test_documents/John_Kerry.txt @@ -0,0 +1,52 @@ +Kerry was born in Aurora, Colorado and attended boarding school in Massachusetts and New Hampshire. He graduated from Yale University class of 1966 with a political science major. Kerry enlisted in the Naval Reserve in 1966, and during 1968–1969 served an abbreviated four-month tour of duty in South Vietnam as officer-in-charge (OIC) of a Swift Boat. For that service, he was awarded combat medals that include the Silver Star Medal, Bronze Star Medal, and three Purple Heart Medals. Securing an early return to the United States, Kerry joined the Vietnam Veterans Against the War organization in which he served as a nationally recognized spokesman and as an outspoken opponent of the Vietnam War. He appeared in the Fulbright Hearings before the Senate Committee on Foreign Affairs where he deemed United States war policy in Vietnam to be the cause of war crimes. +After receiving his J.D. from Boston College Law School, Kerry worked in Massachusetts as an Assistant District Attorney. He served as Lieutenant Governor of Massachusetts under Michael Dukakis from 1983 to 1985 and was elected to the U.S. Senate in 1984 and was sworn in the following January. On the Senate Foreign Relations Committee, he led a series of hearings from 1987 to 1989 which were a precursor to the Iran–Contra affair. Kerry was re-elected to additional terms in 1990, 1996, 2002 and 2008. In 2002, Kerry voted to authorize the President "to use force, if necessary, to disarm Saddam Hussein", but warned that the administration should exhaust its diplomatic avenues before launching war. +In his 2004 presidential campaign, Kerry criticized George W. Bush for the Iraq War. He and his running mate, North Carolina Senator John Edwards, lost the election, finishing 35 electoral votes behind Bush and Vice President Dick Cheney. Kerry returned to the Senate, becoming Chairman of the Senate Committee on Small Business and Entrepreneurship in 2007 and then of the Foreign Relations Committee in 2009. In January 2013, Kerry was nominated by President Barack Obama to succeed outgoing Secretary of State Hillary Clinton and then confirmed by the U.S. Senate, assuming the office on February 1, 2013. +John Forbes Kerry was born on December 11, 1943 in Aurora, Colorado, at Fitzsimons Army Hospital. He was the second oldest of four children born to Richard John Kerry, a Foreign Service officer and lawyer, and Rosemary Isabel Forbes, a nurse and social activist. His father was raised Catholic (John's paternal grandparents were Austro-Hungarian Jewish immigrants who converted to Catholicism) and his mother was Episcopalian. He was raised with an elder sister named Margaret (born 1941), a younger sister named Diana (born 1947) and a younger brother named Cameron (born 1950). The children were raised in their father's faith; John Kerry served as an altar boy. +In his sophomore year, Kerry became the Chairman of the Liberal Party of the Yale Political Union, and a year later he served as President of the Union. Amongst his influential teachers in this period was Professor H. Bradford Westerfield, who was himself a former President of the Political Union. His involvement with the Political Union gave him an opportunity to be involved with important issues of the day, such as the civil rights movement and the New Frontier program. He also became a member of the secretive Skull and Bones Society, and traveled to Switzerland through AIESEC Yale. +On February 18, 1966, Kerry enlisted in the Naval Reserve. He began his active duty military service on August 19, 1966. After completing 16 weeks of Officer Candidate School at the U.S. Naval Training Center in Newport, Rhode Island, Kerry received his officer's commission on December 16, 1966. During the 2004 election, Kerry posted his military records at his website, and permitted reporters to inspect his medical records. In 2005, Kerry released his military and medical records to the representatives of three news organizations, but has not authorized full public access to those records. +During his tour on the guided missile frigate USS Gridley, Kerry requested duty in South Vietnam, listing as his first preference a position as the commander of a Fast Patrol Craft (PCF), also known as a "Swift boat." These 50-foot (15 m) boats have aluminum hulls and have little or no armor, but are heavily armed and rely on speed. "I didn't really want to get involved in the war", Kerry said in a book of Vietnam reminiscences published in 1986. "When I signed up for the swift boats, they had very little to do with the war. They were engaged in coastal patrolling and that's what I thought I was going to be doing." However, his second choice of billet was on a river patrol boat, or "PBR", which at the time was serving a more dangerous duty on the rivers of Vietnam. +During the night of December 2 and early morning of December 3, 1968, Kerry was in charge of a small boat operating near a peninsula north of Cam Ranh Bay together with a Swift boat (PCF-60). According to Kerry and the two crewmen who accompanied him that night, Patrick Runyon and William Zaladonis, they surprised a group of Vietnamese men unloading sampans at a river crossing, who began running and failed to obey an order to stop. As the men fled, Kerry and his crew opened fire on the sampans and destroyed them, then rapidly left. During this encounter, Kerry received a shrapnel wound in the left arm above the elbow. It was for this injury that Kerry received his first Purple Heart Medal. +Kerry received his second Purple Heart for a wound received in action on the Bồ Đề River on February 20, 1969. The plan had been for the Swift boats to be accompanied by support helicopters. On the way up the Bo De, however, the helicopters were attacked. As the Swift boats reached the Cửa Lớn River, Kerry's boat was hit by a B-40 rocket (rocket propelled grenade round), and a piece of shrapnel hit Kerry's left leg, wounding him. Thereafter, enemy fire ceased and his boat reached the Gulf of Thailand safely. Kerry continues to have shrapnel embedded in his left thigh because the doctors that first treated him decided to remove the damaged tissue and close the wound with sutures rather than make a wide opening to remove the shrapnel. Though wounded like several others earlier that day, Kerry did not lose any time off from duty. +Eight days later, on February 28, 1969, came the events for which Kerry was awarded his Silver Star Medal. On this occasion, Kerry was in tactical command of his Swift boat and two other Swift boats during a combat operation. Their mission on the Duong Keo River included bringing an underwater demolition team and dozens of South Vietnamese Marines to destroy enemy sampans, structures and bunkers as described in the story The Death Of PCF 43. Running into heavy small arms fire from the river banks, Kerry "directed the units to turn to the beach and charge the Viet Cong positions" and he "expertly directed" his boat's fire causing the enemy to flee while at the same time coordinating the insertion of the ninety South Vietnamese troops (according to the original medal citation signed by Admiral Zumwalt). Moving a short distance upstream, Kerry's boat was the target of a B-40 rocket round; Kerry charged the enemy positions and as his boat hove to and beached, a Viet Cong ("VC") insurgent armed with a rocket launcher emerged from a spider hole and ran. While the boat's gunner opened fire, wounding the VC in the leg, and while the other boats approached and offered cover fire, Kerry jumped from the boat to pursue the VC insurgent, subsequently killing him and capturing his loaded rocket launcher. +Kerry's commanding officer, Lieutenant Commander George Elliott, stated to Douglas Brinkley in 2003 that he did not know whether to court-martial Kerry for beaching the boat without orders or give him a medal for saving the crew. Elliott recommended Kerry for the Silver Star, and Zumwalt flew into An Thoi to personally award medals to Kerry and the rest of the sailors involved in the mission. The Navy's account of Kerry's actions is presented in the original medal citation signed by Zumwalt. The engagement was documented in an after-action report, a press release written on March 1, 1969, and a historical summary dated March 17, 1969. +On March 13, 1969, on the Bái Háp River, Kerry was in charge of one of five Swift boats that were returning to their base after performing an Operation Sealords mission to transport South Vietnamese troops from the garrison at Cái Nước and MIKE Force advisors for a raid on a Vietcong camp located on the Rach Dong Cung canal. Earlier in the day, Kerry received a slight shrapnel wound in the buttocks from blowing up a rice bunker. Debarking some but not all of the passengers at a small village, the boats approached a fishing weir; one group of boats went around to the left of the weir, hugging the shore, and a group with Kerry's PCF-94 boat went around to the right, along the shoreline. A mine was detonated directly beneath the lead boat, PCF-3, as it crossed the weir to the left, lifting PCF-3 "about 2-3 ft out of water". +James Rassmann, a Green Beret advisor who was aboard Kerry's PCF-94, was knocked overboard when, according to witnesses and the documentation of the event, a mine or rocket exploded close to the boat. According to the documentation for the event, Kerry's arm was injured when he was thrown against a bulkhead during the explosion. PCF 94 returned to the scene and Kerry rescued Rassmann who was receiving sniper fire from the water. Kerry received the Bronze Star Medal with Combat "V" for "heroic achievement", for his actions during this incident; he also received his third Purple Heart. +After Kerry's third qualifying wound, he was entitled per Navy regulations to reassignment away from combat duties. Kerry's preferred choice for reassignment was as a military aide in Boston, New York or Washington, D.C. On April 11, 1969, he reported to the Brooklyn-based Atlantic Military Sea Transportation Service, where he would remain on active duty for the following year as a personal aide to an officer, Rear Admiral Walter Schlech. On January 1, 1970 Kerry was temporarily promoted to full Lieutenant. Kerry had agreed to an extension of his active duty obligation from December 1969 to August 1970 in order to perform Swift Boat duty. John Kerry was on active duty in the United States Navy from August 1966 until January 1970. He continued to serve in the Naval Reserve until February 1978. +With the continuing controversy that had surrounded the military service of George W. Bush since the 2000 Presidential election (when he was accused of having used his father's political influence to gain entrance to the Texas Air National Guard, thereby protecting himself from conscription into the United States Army, and possible service in the Vietnam War), John Kerry's contrasting status as a decorated Vietnam War veteran posed a problem for Bush's re-election campaign, which Republicans sought to counter by calling Kerry's war record into question. As the presidential campaign of 2004 developed, approximately 250 members of a group called Swift Boat Veterans for Truth (SBVT, later renamed Swift Vets and POWs for Truth) opposed Kerry's campaign. The group held press conferences, ran ads and endorsed a book questioning Kerry's service record and his military awards. The group included several members of Kerry's unit, such as Larry Thurlow, who commanded a swift boat alongside of Kerry's, and Stephen Gardner, who served on Kerry's boat. The campaign inspired the widely used political pejorative 'swiftboating', to describe an unfair or untrue political attack. Most of Kerry's former crewmates have stated that SBVT's allegations are false. +After returning to the United States, Kerry joined the Vietnam Veterans Against the War (VVAW). Then numbering about 20,000, VVAW was considered by some (including the administration of President Richard Nixon) to be an effective, if controversial, component of the antiwar movement. Kerry participated in the "Winter Soldier Investigation" conducted by VVAW of U.S. atrocities in Vietnam, and he appears in a film by that name that documents the investigation. According to Nixon Secretary of Defense Melvin Laird, "I didn't approve of what he did, but I understood the protesters quite well", and he declined two requests from the Navy to court martial Reserve Lieutenant Kerry over his antiwar activity. +On April 22, 1971, Kerry appeared before a U.S. Senate committee hearing on proposals relating to ending the war. The day after this testimony, Kerry participated in a demonstration with thousands of other veterans in which he and other Vietnam War veterans threw their medals and service ribbons over a fence erected at the front steps of the United States Capitol building to dramatize their opposition to the war. Jack Smith, a Marine, read a statement explaining why the veterans were returning their military awards to the government. For more than two hours, almost 1000 angry veterans tossed their medals, ribbons, hats, jackets, and military papers over the fence. Each veteran gave his or her name, hometown, branch of service and a statement. Kerry threw some of his own decorations and awards as well as some given to him by other veterans to throw. As Kerry threw his decorations over the fence, his statement was: "I'm not doing this for any violent reasons, but for peace and justice, and to try and make this country wake up once and for all." +Kerry was arrested on May 30, 1971, during a VVAW march to honor American POWs held captive by North Vietnam. The march was planned as a multi-day event from Concord to Boston, and while in Lexington, participants tried to camp on the village green. At 2:30 a.m., local and state police arrested 441 demonstrators, including Kerry, for trespassing. All were given the Miranda Warning and were hauled away on school buses to spend the night at the Lexington Public Works Garage. Kerry and the other protesters later paid a $5 fine, and were released. The mass arrests caused a community backlash and ended up giving positive coverage to the VVAW. +In 1970, Kerry had considered running for Congress in the Democratic primary against hawkish Democrat Philip J. Philbin of Massachusetts's 3rd congressional district, but deferred in favour of Robert Drinan, a Jesuit priest and anti-war activist, who went on to defeat Philbin. In February 1972, Kerry's wife bought a house in Worcester, with Kerry intending to run against the 4th district's ageing thirteen-term incumbent Democrat, Harold Donohue. The couple never moved in. After Republican Congressman F. Bradford Morse of the neighbouring 5th district announced his retirement and then resignation to become Under-Secretary-General for Political and General Assembly Affairs at the United Nations. The couple instead rented an apartment in Lowell, so that Kerry could run to succeed him. +Including Kerry, the Democratic primary race had 10 candidates, including attorney Paul J. Sheehy, State Representative Anthony R. DiFruscia, John J. Desmond and Robert B. Kennedy. Kerry ran a "very expensive, sophisticated campaign", financed by out-of-state backers and supported by many young volunteers. DiFruscia's campaign headquarters shared the same building as Kerry's. On the eve of the September 19 primary, police found Kerry's younger brother Cameron and campaign field director Thomas J. Vallely, breaking into where the building's telephone lines were located. They were arrested and charged with "breaking and entering with the intent to commit grand larceny", but the charges were dropped a year later. At the time of the incident, DiFruscia alleged that the two were trying to disrupt his get-out-the vote efforts. Vallely and Cameron Kerry maintained that they were only checking their own telephone lines because they had received an anonymous call warning that the Kerry lines would be cut. +In the general election, Kerry was initially favored to defeat the Republican candidate, former State Representative Paul W. Cronin, and conservative Democrat Roger P. Durkin, who ran as an Independent. A week after the primary, one poll put Kerry 26-points ahead of Cronin. His campaign called for a national health insurance system, discounted prescription drugs for the unemployed, a jobs programme to clean up the Merrimack River and rent controls in Lowell and Lawrence. A major obstacle, however, was the district's leading newspaper, the conservative The Sun. The paper editorialized against him. It also ran critical news stories about his out-of-state contributions and his "carpetbagging", because he had only moved into the district in April. Subsequently released "Watergate" Oval Office tape recordings of the Nixon White House showed that defeating Kerry's candidacy had attracted the personal attention of President Nixon. Kerry himself asserts that Nixon sent operatives to Lowell to help derail his campaign. +The race was the most expensive for Congress in the country that year and four days before the general election, Durkin withdrew and endorsed Cronin, hoping to see Kerry defeated. The week before, a poll had put Kerry 10 points ahead of Cronin, with Dukin on 13%. In the final days of the campaign, Kerry sensed that it was "slipping away" and Cronin emerged victorious by 110,970 votes (53.45%) to Kerry's 92,847 (44.72%). After his defeat, Kerry lamented in a letter to supporters that "for two solid weeks, [The Sun] called me un-American, New Left antiwar agitator, unpatriotic, and labeled me every other 'un-' and 'anti-' that they could find. It's hard to believe that one newspaper could be so powerful, but they were." He later felt that his failure to respond directly to The Sun's attacks cost him the race. +After Kerry's 1972 defeat, he and his wife bought a house in Belvidere, Lowell, entering a decade which his brother Cameron later called "the years in exile". He spent some time working as a fundraiser for the Cooperative for Assistance and Relief Everywhere (CARE), an international humanitarian organization. In September 1973, he entered Boston College Law School. While studying, Kerry worked as a talk radio host on WBZ and, in July 1974, was named executive director of Mass Action, a Massachusetts advocacy association. +In January 1977, Droney promoted him to First Assistant District Attorney, essentially making Kerry his campaign and media surrogate because Droney was afflicted with amyotrophic lateral sclerosis (ALS, or Lou Gehrig's Disease). As First Assistant, Kerry tried cases, which included winning convictions in a high-profile rape case and a murder. He also played a role in administering the office, including initiating the creation of special white-collar and organized crime units, creating programs to address the problems of rape and other crime victims and witnesses, and managing trial calendars to reflect case priorities. It was in this role in 1978 that Kerry announced an investigation into possible criminal charges against then Senator Edward Brooke, regarding "misstatements" in his first divorce trial. The inquiry ended with no charges being brought after investigators and prosecutors determined that Brooke's misstatements were pertinent to the case, but were not material enough to have affected the outcome. +Droney's health was poor and Kerry had decided to run for his position in the 1978 election should Droney drop out. However, Droney was re-elected and his health improved; he went on to re-assume many of the duties that he had delegated to Kerry. Kerry thus decided to leave, departing in 1979 with assistant DA Roanne Sragow to set up their own law firm. Kerry also worked as a commentator for WCVB-TV and co-founded a bakery, Kilvert & Forbes Ltd., with businessman and former Kennedy aide K. Dun Gifford. +The junior U.S. Senator from Massachusetts, Paul Tsongas, announced in 1984 that he would be stepping down for health reasons. Kerry ran, and as in his 1982 race for Lieutenant Governor, he did not receive the endorsement of the party regulars at the state Democratic convention. Congressman James Shannon, a favorite of House Speaker Tip O'Neill, was the early favorite to win the nomination, and he "won broad establishment support and led in early polling." Again as in 1982, however, Kerry prevailed in a close primary. +On April 18, 1985, a few months after taking his Senate seat, Kerry and Senator Tom Harkin of Iowa traveled to Nicaragua and met the country's president, Daniel Ortega. Though Ortega had won internationally certified elections, the trip was criticized because Ortega and his leftist Sandinista government had strong ties to Cuba and the USSR and were accused of human rights abuses. The Sandinista government was opposed by the right-wing CIA-backed rebels known as the Contras. While in Nicaragua, Kerry and Harkin talked to people on both sides of the conflict. Through the senators, Ortega offered a cease-fire agreement in exchange for the U.S. dropping support of the Contras. The offer was denounced by the Reagan administration as a "propaganda initiative" designed to influence a House vote on a $14 million Contra aid package, but Kerry said "I am willing..... to take the risk in the effort to put to test the good faith of the Sandinistas." The House voted down the Contra aid, but Ortega flew to Moscow to accept a $200 million loan the next day, which in part prompted the House to pass a larger $27 million aid package six weeks later. +Meanwhile, Kerry's staff began their own investigations and, on October 14, issued a report that exposed illegal activities on the part of Lieutenant Colonel Oliver North, who had set up a private network involving the National Security Council and the CIA to deliver military equipment to right-wing Nicaraguan rebels (Contras). In effect, North and certain members of the President's administration were accused by Kerry's report of illegally funding and supplying armed militants without the authorization of Congress. Kerry's staff investigation, based on a year-long inquiry and interviews with fifty unnamed sources, is said to raise "serious questions about whether the United States has abided by the law in its handling of the contras over the past three years." +The Kerry Committee report found that "the Contra drug links included..... payments to drug traffickers by the U.S. State Department of funds authorized by the Congress for humanitarian assistance to the Contras, in some cases after the traffickers had been indicted by federal law enforcement agencies on drug charges, in others while traffickers were under active investigation by these same agencies." The U.S. State Department paid over $806,000 to known drug traffickers to carry humanitarian assistance to the Contras. Kerry's findings provoked little reaction in the media and official Washington. +During their investigation of Noriega, Kerry's staff found reason to believe that the Pakistan-based Bank of Credit and Commerce International (BCCI) had facilitated Noriega's drug trafficking and money laundering. This led to a separate inquiry into BCCI, and as a result, banking regulators shut down BCCI in 1991. In December 1992, Kerry and Senator Hank Brown, a Republican from Colorado, released The BCCI Affair, a report on the BCCI scandal. The report showed that the bank was crooked and was working with terrorists, including Abu Nidal. It blasted the Department of Justice, the Department of the Treasury, the Customs Service, the Federal Reserve Bank, as well as influential lobbyists and the CIA. +In 1996, Kerry faced a difficult re-election fight against Governor William Weld, a popular Republican incumbent who had been re-elected in 1994 with 71% of the vote. The race was covered nationwide as one of the most closely watched Senate races that year. Kerry and Weld held several debates and negotiated a campaign spending cap of $6.9 million at Kerry's Beacon Hill townhouse. Both candidates spent more than the cap, with each camp accusing the other of being first to break the agreement. During the campaign, Kerry spoke briefly at the 1996 Democratic National Convention. Kerry won re-election with 53 percent to Weld's 45 percent. +Kerry said that he had intended the remark as a jab at President Bush, and described the remarks as a "botched joke", having inadvertently left out the key word "us" (which would have been, "If you don't, you get us stuck in Iraq"), as well as leaving the phrase "just ask President Bush" off of the end of the sentence. In Kerry's prepared remarks, which he released during the ensuing media frenzy, the corresponding line was "... you end up getting us stuck in a war in Iraq. Just ask President Bush." He also said that from the context of the speech which, prior to the "stuck in Iraq" line, made several specific references to Bush and elements of his biography, that Kerry was referring to President Bush and not American troops in general. +Kerry "has emerged in the past few years as an important envoy for Afghanistan and Pakistan during times of crisis," a Washington Post report stated in May 2011, as Kerry undertook another trip to the two countries. The killing of Osama bin Laden "has generated perhaps the most important crossroads yet," the report continued, as the senator spoke at a press conference and prepared to fly from Kabul to Pakistan. Among matters discussed during the May visit to Pakistan, under the general rubric of "recalibrating" the bilateral relationship, Kerry sought and retrieved from the Pakistanis the tail-section of the U.S. helicopter which had had to be abandoned at Abbottabad during the bin Laden strike. In 2013, Kerry met with Pakistan's army chief Gen. Ashfaq Parvez Kayani to discuss the peace process with the Taliban in Afghanistan. +Most analyses place Kerry's voting record on the left within the Senate Democratic caucus. During the 2004 presidential election he was portrayed as a staunch liberal by conservative groups and the Bush campaign, who often noted that in 2003 Kerry was rated the National Journal's top Senate liberal. However, that rating was based only upon voting on legislation within that past year. In fact, in terms of career voting records, the National Journal found that Kerry is the 11th most liberal member of the Senate. Most analyses find that Kerry is at least slightly more liberal than the typical Democratic Senator. Kerry has stated that he opposes privatizing Social Security, supports abortion rights for adult women and minors, supports same-sex marriage, opposes capital punishment except for terrorists, supports most gun control laws, and is generally a supporter of trade agreements. Kerry supported the North American Free Trade Agreement and Most Favored Nation status for China, but opposed the Central American Free Trade Agreement.[citation needed] +In the lead up to the Iraq War, Kerry said on October 9, 2002; "I will be voting to give the President of the United States the authority to use force, if necessary, to disarm Saddam Hussein because I believe that a deadly arsenal of weapons of mass destruction in his hands is a real and grave threat to our security." Bush relied on that resolution in ordering the 2003 invasion of Iraq. Kerry also gave a January 23, 2003 speech to Georgetown University saying "Without question, we need to disarm Saddam Hussein. He is a brutal, murderous dictator; leading an oppressive regime he presents a particularly grievous threat because he is so consistently prone to miscalculation. So the threat of Saddam Hussein with weapons of mass destruction is real." Kerry did, however, warn that the administration should exhaust its diplomatic avenues before launching war: "Mr. President, do not rush to war, take the time to build the coalition, because it's not winning the war that's hard, it's winning the peace that's hard." +Kerry chaired the Senate Select Committee on POW/MIA Affairs from 1991 to 1993. The committee's report, which Kerry endorsed, stated there was "no compelling evidence that proves that any American remains alive in captivity in Southeast Asia." In 1994 the Senate passed a resolution, sponsored by Kerry and fellow Vietnam veteran John McCain, that called for an end to the existing trade embargo against Vietnam; it was intended to pave the way for normalization. In 1995, President Bill Clinton normalized diplomatic relations with the country of Vietnam. +In the 2004 Democratic presidential primaries, John Kerry defeated several Democratic rivals, including Sen. John Edwards (D-North Carolina.), former Vermont Governor Howard Dean and retired Army General Wesley Clark. His victory in the Iowa caucuses is widely believed to be the tipping point where Kerry revived his sagging campaign in New Hampshire and the February 3, 2004, primary states like Arizona, South Carolina and New Mexico. Kerry then went on to win landslide victories in Nevada and Wisconsin. Kerry thus won the Democratic nomination to run for President of the United States against incumbent George W. Bush. On July 6, 2004, he announced his selection of John Edwards as his running mate. Democratic strategist Bob Shrum, who was Kerry's 2004 campaign adviser, wrote an article in Time magazine claiming that after the election, Kerry had said that he wished he'd never picked Edwards, and that the two have since stopped speaking to each other. In a subsequent appearance on ABC's This Week, Kerry refused to respond to Shrum's allegation, calling it a "ridiculous waste of time." +During his bid to be elected president in 2004, Kerry frequently criticized President George W. Bush for the Iraq War. While Kerry had initially voted in support of authorizing President Bush to use force in dealing with Saddam Hussein, he voted against an $87 billion supplemental appropriations bill to pay for the subsequent war. His statement on March 16, 2004, "I actually did vote for the $87 billion before I voted against it," helped the Bush campaign to paint him as a flip-flopper and has been cited as contributing to Kerry's defeat. +Kerry established a separate political action committee, Keeping America's Promise, which declared as its mandate "A Democratic Congress will restore accountability to Washington and help change a disastrous course in Iraq", and raised money and channeled contributions to Democratic candidates in state and federal races. Through Keeping America's Promise in 2005, Kerry raised over $5.5 million for other Democrats up and down the ballot. Through his campaign account and his political action committee, the Kerry campaign operation generated more than $10 million for various party committees and 179 candidates for the U.S. House, Senate, state and local offices in 42 states focusing on the midterm elections during the 2006 election cycle. "Cumulatively, John Kerry has done as much if not more than any other individual senator", Hassan Nemazee, the national finance chairman of the DSCC said. +On December 15, 2012, several news outlets reported that President Barack Obama would nominate Kerry to succeed Hillary Clinton as Secretary of State, after Susan Rice, widely seen as Obama's preferred choice, withdrew her name from consideration citing a politicized confirmation process following criticism of her response to the 2012 Benghazi attack. On December 21, Obama proposed the nomination which received positive commentary. His confirmation hearing took place on January 24, 2013, before the Senate Foreign Relations Committee, the same panel where he first testified in 1971. The committee unanimously voted to approve him on January 29, 2013, and the same day the full Senate confirmed him on a vote of 94–3. In a letter to Massachusetts Governor Deval Patrick, Kerry announced his resignation from the Senate effective February 1. +In the State Department, Kerry quickly earned a reputation "for being aloof, keeping to himself, and not bothering to read staff memos." Career State Department officials have complained that power has become too centralized under Kerry's leadership, which slows department operations when Kerry is on one of his frequent overseas trips. Others in State describe Kerry as having "a kind of diplomatic attention deficit disorder" as he shifts from topic to topic instead of focusing on long-term strategy. When asked whether he was traveling too much, he responded, "Hell no. I'm not slowing down." Despite Kerry's early achievements, morale at State is lower than under Hillary Clinton according to department employees. However, after Kerry's first six months in the State Department, a Gallup poll found he had high approval ratings among Americans as Secretary of State. After a year, another poll showed Kerry's favorability continued to rise. Less than two years into Kerry's term, the Foreign Policy Magazine's 2014 Ivory Tower survey of international relations scholars asked, "Who was the most effective U.S. Secretary of State in the past 50 years?"; John Kerry and Lawrence Eagleburger tied for 11th place out of the 15 confirmed Secretaries of State in that period. +In January 2014, having met with Secretary of State, Archbishop Pietro Parolin, Kerry said "We touched on just about every major issue that we are both working on, that are issues of concern to all of us. First of all, we talked at great length about Syria, and I was particularly appreciative for the Archbishop’s raising this issue, and equally grateful for the Holy Father’s comments – the Pope's comments yesterday regarding his support for the Geneva II process. We welcome that support. It is very important to have broad support, and I know that the Pope is particularly concerned about the massive numbers of displaced human beings and the violence that has taken over 130,000 lives". +Kerry said on September 9 in response to a reporter's question about whether Syrian President Bashar al-Assad could avert a military strike: "He could turn over every single bit of his chemical weapons to the international community in the next week. Turn it over, all of it, without delay, and allow a full and total accounting for that. But he isn't about to do it, and it can't be done, obviously." This unscripted remark initiated a process that would lead to Syria agreeing to relinquish and destroy its chemical weapons arsenal, as Russia treated Kerry's statement as a serious proposal. Russian Foreign Minister Sergey Lavrov said Russia would work "immediately" to convince Syria relinquish and destroy its large chemical weapons arsenal. Syria quickly welcomed this proposal and on September 14, the UN formally accepted Syria's application to join the convention banning chemical weapons, and separately, the U.S. and Russia agreed on a plan to eliminate Syria's chemical weapons by the middle of 2014. On September 28, the UN Security Council passed a resolution ordering the destruction of Syria's chemical weapons and condemning the August 21 Ghouta attack. +In a speech before the Organization of American States in November 2013, Kerry remarked that the era of the Monroe Doctrine was over. He went on to explain, "The relationship that we seek and that we have worked hard to foster is not about a United States declaration about how and when it will intervene in the affairs of other American states. It's about all of our countries viewing one another as equals, sharing responsibilities, cooperating on security issues, and adhering not to doctrine, but to the decisions that we make as partners to advance the values and the interests that we share." +Kerry's paternal grandparents, shoe businessman Frederick A. "Fred" Kerry and musician Ida Lowe, were immigrants from the Austro-Hungarian Empire. Fred was born as "Fritz Kohn" before he and Ida took on the "Kerry" name and moved to the United States. Fred and Ida were born Jewish, and converted to Catholicism together in Austria. His maternal ancestors were of Scottish and English descent, and his maternal grandfather James Grant Forbes II was a member of the Forbes family, while his maternal grandmother Margaret Tyndal Winthrop was a member of the Dudley–Winthrop family. Margaret's paternal grandfather Robert Charles Winthrop served as the 22nd Speaker of the U.S. House of Representatives. Robert's father was Governor Thomas Lindall Winthrop. Thomas' father John Still Winthrop was a great-great-grandson of Massachusetts Bay Colony Governor John Winthrop and great-grandson of Governor Thomas Dudley. Through his mother, John is a first cousin once removed of French politician Brice Lalonde. +Alexandra was born days before Kerry began law school. In 1982, Julia asked Kerry for a separation while she was suffering from severe depression. They were divorced on July 25, 1988, and the marriage was formally annulled in 1997. "After 14 years as a political wife, I associated politics only with anger, fear and loneliness" she wrote in A Change of Heart, her book about depression. Thorne later married Richard Charlesworth, an architect, and moved to Bozeman, Montana, where she became active in local environmental groups such as the Greater Yellowstone Coalition. Thorne supported Kerry's 2004 presidential run. She died of cancer on April 27, 2006. +Kerry and his second wife, Mozambican-born businesswoman and philanthropist Maria Teresa Thierstein Simões Ferreira (known as Teresa), the widow of Kerry's late Pennsylvania Republican Senate colleague Henry John Heinz III, were introduced to each other by Heinz at an Earth Day rally in 1990. Early the following year, Senator Heinz was killed in a plane crash near Lower Merion. Teresa has three sons from her previous marriage to Heinz, Henry John Heinz IV, André Thierstein Heinz, and Christopher Drake Heinz. Heinz and Kerry were married on May 26, 1995, in Nantucket, Massachusetts. +The Forbes 400 survey estimated in 2004 that Teresa Heinz Kerry had a net worth of $750 million. However, estimates have frequently varied, ranging from around $165 million to as high as $3.2 billion, according to a study in the Los Angeles Times. Regardless of which figure is correct, Kerry was the wealthiest U.S. Senator while serving in the Senate. Independent of Heinz, Kerry is wealthy in his own right, and is the beneficiary of at least four trusts inherited from Forbes family relatives, including his mother, Rosemary Forbes Kerry, who died in 2002. Forbes magazine (named for the Forbes family of publishers, unrelated to Kerry) estimated that if elected, and if Heinz family assets were included, Kerry would have been the third-richest U.S. President in history, when adjusted for inflation. This assessment was based on Heinz and Kerry's combined assets, but the couple signed a prenuptial agreement that keeps their assets separate. Kerry's financial disclosure form for 2011 put his personal assets in the range of $230,000,000 to $320,000,000, including the assets of his spouse and any dependent children. This included slightly more than three million dollars worth of H. J. Heinz Company assets, which increased in value by over six hundred thousand dollars in 2013 when Berkshire Hathaway announced their intention to purchase the company. +Kerry is a Roman Catholic, and is said to carry a religious rosary, a prayer book, and a St. Christopher medal (the patron saint of travelers) when he campaigned. While Kerry is personally against abortion, he supports a woman's legal right to have one. Discussing his faith, Kerry said, "I thought of being a priest. I was very religious while at school in Switzerland. I was an altar boy and prayed all the time. I was very centered around the Mass and the church." He also said that the Letters of Paul (Apostle Paul) moved him the most, stating that they taught him to "not feel sorry for myself." +Kerry told Christianity Today in October 2004 "I'm a Catholic and I practice, but at the same time I have an open-mindedness to many other expressions of spirituality that come through different religions... I've spent some time reading and thinking about religion and trying to study it, and I've arrived at not so much a sense of the differences, but a sense of the similarities in so many ways." He said that he believed that the Torah, the Qur'an, and the Bible all share a fundamental story which connects with readers. +In addition to the sports he played at Yale, Kerry is described by Sports Illustrated, among others, as an "avid cyclist", primarily riding on a road bike. Prior to his presidential bid, Kerry was known to have participated in several long-distance rides (centuries). Even during his many campaigns, he was reported to have visited bicycle stores in both his home state and elsewhere. His staff requested recumbent stationary bikes for his hotel rooms. He has also been a snowboarder, windsurfer, and sailor. +According to the Boston Herald, dated July 23, 2010, Kerry commissioned construction on a new $7 million yacht (a Friendship 75) in New Zealand and moored it in Portsmouth, Rhode Island, where the Friendship yacht company is based. The article claimed this allowed him to avoid paying Massachusetts taxes on the property including approximately $437,500 in sales tax and an annual excise tax of about $500. However, on July 27, 2010, Kerry stated he had yet to take legal possession of the boat, had not intended to avoid the taxes, and that when he took possession, he would pay the taxes whether he owed them or not. diff --git a/e2e/samples/test_documents/Late_Middle_Ages.txt b/e2e/samples/test_documents/Late_Middle_Ages.txt new file mode 100644 index 0000000000..c5fc31f914 --- /dev/null +++ b/e2e/samples/test_documents/Late_Middle_Ages.txt @@ -0,0 +1,40 @@ +Around 1300, centuries of prosperity and growth in Europe came to a halt. A series of famines and plagues, including the Great Famine of 1315–1317 and the Black Death, reduced the population to around half of what it was before the calamities. Along with depopulation came social unrest and endemic warfare. France and England experienced serious peasant uprisings, such as the Jacquerie and the Peasants' Revolt, as well as over a century of intermittent conflict in the Hundred Years' War. To add to the many problems of the period, the unity of the Catholic Church was shattered by the Western Schism. Collectively these events are sometimes called the Crisis of the Late Middle Ages. +Despite these crises, the 14th century was also a time of great progress in the arts and sciences. Following a renewed interest in ancient Greek and Roman texts that took root in the High Middle Ages, the Italian Renaissance began. The absorption of Latin texts had started before the Renaissance of the 12th century through contact with Arabs during the Crusades, but the availability of important Greek texts accelerated with the capture of Constantinople by the Ottoman Turks, when many Byzantine scholars had to seek refuge in the West, particularly Italy. +Combined with this influx of classical ideas was the invention of printing which facilitated dissemination of the printed word and democratized learning. These two things would later lead to the Protestant Reformation. Toward the end of the period, an era of discovery began (Age of Discovery). The rise of the Ottoman Empire, culminating in the Fall of Constantinople in 1453, eroded the last remnants of the Byzantine Empire and cut off trading possibilities with the east. Europeans were forced to seek new trading routes, leading to the expedition of Columbus to the Americas in 1492, and Vasco da Gama’s circumnavigation of India and Africa in 1498. Their discoveries strengthened the economy and power of European nations. +The term "Late Middle Ages" refers to one of the three periods of the Middle Ages, along with the Early Middle Ages and the High Middle Ages. Leonardo Bruni was the first historian to use tripartite periodization in his History of the Florentine People (1442). Flavio Biondo used a similar framework in Decades of History from the Deterioration of the Roman Empire (1439–1453). Tripartite periodization became standard after the German historian Christoph Cellarius published Universal History Divided into an Ancient, Medieval, and New Period (1683). +As economic and demographic methods were applied to the study of history, the trend was increasingly to see the late Middle Ages as a period of recession and crisis. Belgian historian Henri Pirenne continued the subdivision of Early, High, and Late Middle Ages in the years around World War I. Yet it was his Dutch colleague, Johan Huizinga, who was primarily responsible for popularising the pessimistic view of the Late Middle Ages, with his book The Autumn of the Middle Ages (1919). To Huizinga, whose research focused on France and the Low Countries rather than Italy, despair and decline were the main themes, not rebirth. +Modern historiography on the period has reached a consensus between the two extremes of innovation and crisis. It is now (generally) acknowledged that conditions were vastly different north and south of the Alps, and "Late Middle Ages" is often avoided entirely within Italian historiography. The term "Renaissance" is still considered useful for describing certain intellectual, cultural, or artistic developments, but not as the defining feature of an entire European historical epoch. The period from the early 14th century up until – and sometimes including – the 16th century, is rather seen as characterised by other trends: demographic and economic decline followed by recovery, the end of western religious unity and the subsequent emergence of the nation state, and the expansion of European influence onto the rest of the world. +After the failed union of Sweden and Norway of 1319–1365, the pan-Scandinavian Kalmar Union was instituted in 1397. The Swedes were reluctant members of the Danish-dominated union from the start. In an attempt to subdue the Swedes, King Christian II of Denmark had large numbers of the Swedish aristocracy killed in the Stockholm Bloodbath of 1520. Yet this measure only led to further hostilities, and Sweden broke away for good in 1523. Norway, on the other hand, became an inferior party of the union and remained united with Denmark until 1814. +Bohemia prospered in the 14th century, and the Golden Bull of 1356 made the king of Bohemia first among the imperial electors, but the Hussite revolution threw the country into crisis. The Holy Roman Empire passed to the Habsburgs in 1438, where it remained until its dissolution in 1806. Yet in spite of the extensive territories held by the Habsburgs, the Empire itself remained fragmented, and much real power and influence lay with the individual principalities. In addition, financial institutions, such as the Hanseatic League and the Fugger family, held great power, on both economic and a political levels. +Louis did not leave a son as heir after his death in 1382. Instead, he named as his heir the young prince Sigismund of Luxemburg, who was 11 years old. The Hungarian nobility did not accept his claim, and the result was an internal war. Sigismund eventually achieved total control of Hungary and established his court in Buda and Visegrád. Both palaces were rebuilt and improved, and were considered the richest of the time in Europe. Inheriting the throne of Bohemia and the Holy Roman Empire, Sigismund continued conducting his politics from Hungary, but he was kept busy fighting the Hussites and the Ottoman Empire, which was becoming a menace to Europe in the beginning of the 15th century. +The Bulgarian Empire was in decline by the 14th century, and the ascendancy of Serbia was marked by the Serbian victory over the Bulgarians in the Battle of Velbazhd in 1330. By 1346, the Serbian king Stefan Dušan had been proclaimed emperor. Yet Serbian dominance was short-lived; the Serbian army led by the Lazar Hrebljevanovic was defeated by the Ottomans at the Battle of Kosovo in 1389, where most of the Serbian nobility was killed and the south of the country came under Ottoman occupation, as much of southern Bulgaria had become Ottoman territory in 1371. Northern remnants of Bulgaria were finally conquered by 1396, Serbia fell in 1459, Bosnia in 1463, and Albania was finally subordinated in 1479 only a few years after the death of Skanderbeg. Belgrade, an Hungarian domain at the time, was the last large Balkan city to fall under Ottoman rule, in 1521. By the end of the medieval period, the entire Balkan peninsula was annexed by, or became vassal to, the Ottomans. +Avignon was the seat of the papacy from 1309 to 1376. With the return of the Pope to Rome in 1378, the Papal State developed into a major secular power, culminating in the morally corrupt papacy of Alexander VI. Florence grew to prominence amongst the Italian city-states through financial business, and the dominant Medici family became important promoters of the Renaissance through their patronage of the arts. Other city states in northern Italy also expanded their territories and consolidated their power, primarily Milan and Venice. The War of the Sicilian Vespers had by the early 14th century divided southern Italy into an Aragon Kingdom of Sicily and an Anjou Kingdom of Naples. In 1442, the two kingdoms were effectively united under Aragonese control. +The 1469 marriage of Isabella I of Castile and Ferdinand II of Aragon and the 1479 death of John II of Aragon led to the creation of modern-day Spain. In 1492, Granada was captured from the Moors, thereby completing the Reconquista. Portugal had during the 15th century – particularly under Henry the Navigator – gradually explored the coast of Africa, and in 1498, Vasco da Gama found the sea route to India. The Spanish monarchs met the Portuguese challenge by financing the expedition of Christopher Columbus to find a western sea route to India, leading to the discovery of the Americas in 1492. +Around 1300–1350 the Medieval Warm Period gave way to the Little Ice Age. The colder climate resulted in agricultural crises, the first of which is known as the Great Famine of 1315-1317. The demographic consequences of this famine, however, were not as severe as the plagues that occurred later in the century, particularly the Black Death. Estimates of the death rate caused by this epidemic range from one third to as much as sixty percent. By around 1420, the accumulated effect of recurring plagues and famines had reduced the population of Europe to perhaps no more than a third of what it was a century earlier. The effects of natural disasters were exacerbated by armed conflicts; this was particularly the case in France during the Hundred Years' War. +As the European population was severely reduced, land became more plentiful for the survivors, and labour consequently more expensive. Attempts by landowners to forcibly reduce wages, such as the English 1351 Statute of Laborers, were doomed to fail. These efforts resulted in nothing more than fostering resentment among the peasantry, leading to rebellions such as the French Jacquerie in 1358 and the English Peasants' Revolt in 1381. The long-term effect was the virtual end of serfdom in Western Europe. In Eastern Europe, on the other hand, landowners were able to exploit the situation to force the peasantry into even more repressive bondage. +Up until the mid-14th century, Europe had experienced steadily increasing urbanisation. Cities were also decimated by the Black Death, but the role of urban areas as centres of learning, commerce and government ensured continued growth. By 1500, Venice, Milan, Naples, Paris and Constantinople each probably had more than 100,000 inhabitants. Twenty-two other cities were larger than 40,000; most of these were in Italy and the Iberian peninsula, but there were also some in France, the Empire, the Low Countries, plus London in England. +Changes also took place within the recruitment and composition of armies. The use of the national or feudal levy was gradually replaced by paid troops of domestic retinues or foreign mercenaries. The practice was associated with Edward III of England and the condottieri of the Italian city-states. All over Europe, Swiss soldiers were in particularly high demand. At the same time, the period also saw the emergence of the first permanent armies. It was in Valois France, under the heavy demands of the Hundred Years' War, that the armed forces gradually assumed a permanent nature. +Parallel to the military developments emerged also a constantly more elaborate chivalric code of conduct for the warrior class. This new-found ethos can be seen as a response to the diminishing military role of the aristocracy, and gradually it became almost entirely detached from its military origin. The spirit of chivalry was given expression through the new (secular) type of chivalric orders; the first of these was the Order of St. George, founded by Charles I of Hungary in 1325, while the best known was probably the English Order of the Garter, founded by Edward III in 1348. +The French crown's increasing dominance over the Papacy culminated in the transference of the Holy See to Avignon in 1309. When the Pope returned to Rome in 1377, this led to the election of different popes in Avignon and Rome, resulting in the Papal Schism (1378–1417). The Schism divided Europe along political lines; while France, her ally Scotland and the Spanish kingdoms supported the Avignon Papacy, France's enemy England stood behind the Pope in Rome, together with Portugal, Scandinavia and most of the German princes. +Though many of the events were outside the traditional time-period of the Middle Ages, the end of the unity of the Western Church (the Protestant Reformation), was one of the distinguishing characteristics of the medieval period. The Catholic Church had long fought against heretic movements, but during the Late Middle Ages, it started to experience demands for reform from within. The first of these came from Oxford professor John Wycliffe in England. Wycliffe held that the Bible should be the only authority in religious questions, and he spoke out against transubstantiation, celibacy and indulgences. In spite of influential supporters among the English aristocracy, such as John of Gaunt, the movement was not allowed to survive. Though Wycliffe himself was left unmolested, his supporters, the Lollards, were eventually suppressed in England. +The marriage of Richard II of England to Anne of Bohemia established contacts between the two nations and brought Lollard ideas to her homeland. The teachings of the Czech priest Jan Hus were based on those of John Wycliffe, yet his followers, the Hussites, were to have a much greater political impact than the Lollards. Hus gained a great following in Bohemia, and in 1414, he was requested to appear at the Council of Constance to defend his cause. When he was burned as a heretic in 1415, it caused a popular uprising in the Czech lands. The subsequent Hussite Wars fell apart due to internal quarrels and did not result in religious or national independence for the Czechs, but both the Catholic Church and the German element within the country were weakened. +Martin Luther, a German monk, started the German Reformation by posting 95 theses on the castle church of Wittenberg on October 31, 1517. The immediate provocation spurring this act was Pope Leo X’s renewal of the indulgence for the building of the new St. Peter's Basilica in 1514. Luther was challenged to recant his heresy at the Diet of Worms in 1521. When he refused, he was placed under the ban of the Empire by Charles V. Receiving the protection of Frederick the Wise, he was then able to translate the Bible into German. +In the late 13th and early 14th centuries, a process took place – primarily in Italy but partly also in the Empire – that historians have termed a 'commercial revolution'. Among the innovations of the period were new forms of partnership and the issuing of insurance, both of which contributed to reducing the risk of commercial ventures; the bill of exchange and other forms of credit that circumvented the canonical laws for gentiles against usury, and eliminated the dangers of carrying bullion; and new forms of accounting, in particular double-entry bookkeeping, which allowed for better oversight and accuracy. +With the financial expansion, trading rights became more jealously guarded by the commercial elite. Towns saw the growing power of guilds, while on a national level special companies would be granted monopolies on particular trades, like the English wool Staple. The beneficiaries of these developments would accumulate immense wealth. Families like the Fuggers in Germany, the Medicis in Italy, the de la Poles in England, and individuals like Jacques Coeur in France would help finance the wars of kings, and achieve great political influence in the process. +Though there is no doubt that the demographic crisis of the 14th century caused a dramatic fall in production and commerce in absolute terms, there has been a vigorous historical debate over whether the decline was greater than the fall in population. While the older orthodoxy held that the artistic output of the Renaissance was a result of greater opulence, more recent studies have suggested that there might have been a so-called 'depression of the Renaissance'. In spite of convincing arguments for the case, the statistical evidence is simply too incomplete for a definite conclusion to be made. +The predominant school of thought in the 13th century was the Thomistic reconciliation of the teachings of Aristotle with Christian theology. The Condemnation of 1277, enacted at the University of Paris, placed restrictions on ideas that could be interpreted as heretical; restrictions that had implication for Aristotelian thought. An alternative was presented by William of Ockham, who insisted that the world of reason and the world of faith had to be kept apart. Ockham introduced the principle of parsimony – or Occam's razor – whereby a simple theory is preferred to a more complex one, and speculation on unobservable phenomena is avoided. +This new approach liberated scientific speculation from the dogmatic restraints of Aristotelian science, and paved the way for new approaches. Particularly within the field of theories of motion great advances were made, when such scholars as Jean Buridan, Nicole Oresme and the Oxford Calculators challenged the work of Aristotle. Buridan developed the theory of impetus as the cause of the motion of projectiles, which was an important step towards the modern concept of inertia. The works of these scholars anticipated the heliocentric worldview of Nicolaus Copernicus. +Certain technological inventions of the period – whether of Arab or Chinese origin, or unique European innovations – were to have great influence on political and social developments, in particular gunpowder, the printing press and the compass. The introduction of gunpowder to the field of battle affected not only military organisation, but helped advance the nation state. Gutenberg's movable type printing press made possible not only the Reformation, but also a dissemination of knowledge that would lead to a gradually more egalitarian society. The compass, along with other innovations such as the cross-staff, the mariner's astrolabe, and advances in shipbuilding, enabled the navigation of the World Oceans, and the early phases of colonialism. Other inventions had a greater impact on everyday life, such as eyeglasses and the weight-driven clock. +The period saw several important technical innovations, like the principle of linear perspective found in the work of Masaccio, and later described by Brunelleschi. Greater realism was also achieved through the scientific study of anatomy, championed by artists like Donatello. This can be seen particularly well in his sculptures, inspired by the study of classical models. As the centre of the movement shifted to Rome, the period culminated in the High Renaissance masters da Vinci, Michelangelo and Raphael. +The ideas of the Italian Renaissance were slow to cross the Alps into northern Europe, but important artistic innovations were made also in the Low Countries. Though not – as previously believed – the inventor of oil painting, Jan van Eyck was a champion of the new medium, and used it to create works of great realism and minute detail. The two cultures influenced each other and learned from each other, but painting in the Netherlands remained more focused on textures and surfaces than the idealized compositions of Italy. +Dante Alighieri's Divine Comedy, written in the early 14th century, merged a medieval world view with classical ideals. Another promoter of the Italian language was Boccaccio with his Decameron. The application of the vernacular did not entail a rejection of Latin, and both Dante and Boccaccio wrote prolifically in Latin as well as Italian, as would Petrarch later (whose Canzoniere also promoted the vernacular and whose contents are considered the first modern lyric poems). Together the three poets established the Tuscan dialect as the norm for the modern Italian language. +Music was an important part of both secular and spiritual culture, and in the universities it made up part of the quadrivium of the liberal arts. From the early 13th century, the dominant sacred musical form had been the motet; a composition with text in several parts. From the 1330s and onwards, emerged the polyphonic style, which was a more complex fusion of independent voices. Polyphony had been common in the secular music of the Provençal troubadours. Many of these had fallen victim to the 13th-century Albigensian Crusade, but their influence reached the papal court at Avignon. +The main representatives of the new style, often referred to as ars nova as opposed to the ars antiqua, were the composers Philippe de Vitry and Guillaume de Machaut. In Italy, where the Provençal troubadours had also found refuge, the corresponding period goes under the name of trecento, and the leading composers were Giovanni da Cascia, Jacopo da Bologna and Francesco Landini. Prominent reformer of Orthodox Church music from the first half of 14th century was John Kukuzelis; he also introduced a system of notation widely used in the Balkans in the following centuries. +Morality plays emerged as a distinct dramatic form around 1400 and flourished until 1550. The most interesting morality play is The Castle of Perseverance which depicts mankind's progress from birth to death. However, the most famous morality play and perhaps best known medieval drama is Everyman. Everyman receives Death's summons, struggles to escape and finally resigns himself to necessity. Along the way, he is deserted by Kindred, Goods, and Fellowship - only Good Deeds goes with him to the grave. +At the end of the Late Middle Ages, professional actors began to appear in England and Europe. Richard III and Henry VII both maintained small companies of professional actors. Their plays were performed in the Great Hall of a nobleman's residence, often with a raised platform at one end for the audience and a "screen" at the other for the actors. Also important were Mummers' plays, performed during the Christmas season, and court masques. These masques were especially popular during the reign of Henry VIII who had a House of Revels built and an Office of Revels established in 1545. +The end of medieval drama came about due to a number of factors, including the weakening power of the Catholic Church, the Protestant Reformation and the banning of religious plays in many countries. Elizabeth I forbid all religious plays in 1558 and the great cycle plays had been silenced by the 1580s. Similarly, religious plays were banned in the Netherlands in 1539, the Papal States in 1547 and in Paris in 1548. The abandonment of these plays destroyed the international theatre that had thereto existed and forced each country to develop its own form of drama. It also allowed dramatists to turn to secular subjects and the reviving interest in Greek and Roman theatre provided them with the perfect opportunity. +After the end of the late Middle Ages period, the Renaissance would spread unevenly over continental Europe from the southern European region. The intellectual transformation of the Renaissance is viewed as a bridge between the Middle Ages and the Modern era. Europeans would later begin an era of world discovery. Combined with the influx of classical ideas was the invention of printing which facilitated dissemination of the printed word and democratized learning. These two things would lead to the Protestant Reformation. Europeans also discovered new trading routes, as was the case with Columbus’s travel to the Americas in 1492, and Vasco da Gama’s circumnavigation of Africa and India in 1498. Their discoveries strengthened the economy and power of European nations. +At the end of the 15th century the Ottoman Empire advanced all over Southeastern Europe, eventually conquering the Byzantine Empire and extending control over the Balkan states. Hungary was the last bastion of the Latin Christian world in the East, and fought to keep its rule over a period of two centuries. After the tragic death of the young king Vladislaus I of Hungary during the Battle of Varna in 1444 against the Ottomans, the Kingdom was placed in the hands of count John Hunyadi, who became Hungary's regent-governor (1446–1453). Hunyadi was considered one of the most relevant military figures of the 15th century: Pope Pius II awarded him the title of Athleta Christi or Champion of Christ for being the only hope of resisting the Ottomans from advancing to Central and Western Europe. +Hunyadi succeeded during the Siege of Belgrade in 1456 against the Ottomans, the biggest victory against that empire in decades. This battle became a real Crusade against the Muslims, as the peasants were motivated by the Franciscan monk Saint John of Capistrano, who came from Italy predicating Holy War. The effect that it created in that time was one of the main factors that helped in achieving the victory. However the premature death of the Hungarian Lord left Pannonia defenseless and in chaos. In an extremely unusual event for the Middle Ages, Hunyadi's son, Matthias, was elected as King of Hungary by the nobility. For the first time, a member of an aristocratic family (and not from a royal family) was crowned. +King Matthias Corvinus of Hungary (1458–1490) was one of the most prominent figures of the period, directing campaigns to the West, conquering Bohemia in answer to the Pope's call for help against the Hussite Protestants. Also, in resolving political hostilities with the German emperor Frederick III of Habsburg, he invaded his western domains. Matthew organized the Black Army of mercenary soldiers; it was considered as the biggest army of its time. Using this powerful tool, the Hungarian king led wars against the Turkish armies and stopped the Ottomans during his reign. After the death of Matthew, and with end of the Black Army, the Ottoman Empire grew in strength and Central Europe was defenseless. At the Battle of Mohács, the forces of the Ottoman Empire annihilated the Hungarian army and Louis II of Hungary drowned in the Csele Creek while trying to escape. The leader of the Hungarian army, Pál Tomori, also died in the battle. This is considered to be one of the final battles of Medieval times. +The changes brought about by these developments have led many scholars to view this period as the end of the Middle Ages and beginning of modern history and early modern Europe. However, the division is somewhat artificial, since ancient learning was never entirely absent from European society. As a result there was developmental continuity between the ancient age (via classical antiquity) and the modern age. Some historians, particularly in Italy, prefer not to speak of the late Middle Ages at all, but rather see the high period of the Middle Ages transitioning to the Renaissance and the modern era. diff --git a/e2e/samples/test_documents/Materialism.txt b/e2e/samples/test_documents/Materialism.txt new file mode 100644 index 0000000000..5d5ff8a3de --- /dev/null +++ b/e2e/samples/test_documents/Materialism.txt @@ -0,0 +1,27 @@ +Materialism is a form of philosophical monism which holds that matter is the fundamental substance in nature, and that all phenomena, including mental phenomena and consciousness, are identical with material interactions. +Materialism is closely related to physicalism, the view that all that exists is ultimately physical. Philosophical physicalism has evolved from materialism with the discoveries of the physical sciences to incorporate more sophisticated notions of physicality than mere ordinary matter, such as: spacetime, physical energies and forces, dark matter, and so on. Thus the term "physicalism" is preferred over "materialism" by some, while others use the terms as if they are synonymous. +Materialism belongs to the class of monist ontology. As such, it is different from ontological theories based on dualism or pluralism. For singular explanations of the phenomenal reality, materialism would be in contrast to idealism, neutral monism, and spiritualism. +Despite the large number of philosophical schools and subtle nuances between many, all philosophies are said to fall into one of two primary categories, which are defined in contrast to each other: Idealism, and materialism.[a] The basic proposition of these two categories pertains to the nature of reality, and the primary distinction between them is the way they answer two fundamental questions: "what does reality consist of?" and "how does it originate?" To idealists, spirit or mind or the objects of mind (ideas) are primary, and matter secondary. To materialists, matter is primary, and mind or spirit or ideas are secondary, the product of matter acting upon matter. +The materialist view is perhaps best understood in its opposition to the doctrines of immaterial substance applied to the mind historically, famously by René Descartes. However, by itself materialism says nothing about how material substance should be characterized. In practice, it is frequently assimilated to one variety of physicalism or another. +During the 19th century, Karl Marx and Friedrich Engels extended the concept of materialism to elaborate a materialist conception of history centered on the roughly empirical world of human activity (practice, including labor) and the institutions created, reproduced, or destroyed by that activity (see materialist conception of history). Later Marxists developed the notion of dialectical materialism which characterized later Marxist philosophy and method. +Materialism developed, possibly independently, in several geographically separated regions of Eurasia during what Karl Jaspers termed the Axial Age (approximately 800 to 200 BC). +In Ancient Indian philosophy, materialism developed around 600 BC with the works of Ajita Kesakambali, Payasi, Kanada, and the proponents of the Cārvāka school of philosophy. Kanada became one of the early proponents of atomism. The Nyaya–Vaisesika school (600 BC - 100 BC) developed one of the earliest forms of atomism, though their proofs of God and their positing that the consciousness was not material precludes labelling them as materialists. Buddhist atomism and the Jaina school continued the atomic tradition. +Materialism is often associated with reductionism, according to which the objects or phenomena individuated at one level of description, if they are genuine, must be explicable in terms of the objects or phenomena at some other level of description — typically, at a more reduced level. Non-reductive materialism explicitly rejects this notion, however, taking the material constitution of all particulars to be consistent with the existence of real objects, properties, or phenomena not explicable in the terms canonically used for the basic material constituents. Jerry Fodor influentially argues this view, according to which empirical laws and explanations in "special sciences" like psychology or geology are invisible from the perspective of basic physics. A lot of vigorous literature has grown up around the relation between these views. +Ancient Greek philosophers like Thales, Anaxagoras (ca. 500 BC – 428 BC), Epicurus and Democritus prefigure later materialists. The Latin poem De Rerum Natura by Lucretius (ca. 99 BC – ca. 55 BC) reflects the mechanistic philosophy of Democritus and Epicurus. According to this view, all that exists is matter and void, and all phenomena result from different motions and conglomerations of base material particles called "atoms" (literally: "indivisibles"). De Rerum Natura provides mechanistic explanations for phenomena such as erosion, evaporation, wind, and sound. Famous principles like "nothing can touch body but body" first appeared in the works of Lucretius. Democritus and Epicurus however did not hold to a monist ontology since they held to the ontological separation of matter and space i.e. space being "another kind" of being, indicating that the definition of "materialism" is wider than given scope for in this article. +Later Indian materialist Jayaraashi Bhatta (6th century) in his work Tattvopaplavasimha ("The upsetting of all principles") refuted the Nyaya Sutra epistemology. The materialistic Cārvāka philosophy appears to have died out some time after 1400. When Madhavacharya compiled Sarva-darśana-samgraha (a digest of all philosophies) in the 14th century, he had no Cārvāka/Lokāyata text to quote from, or even refer to. +In early 12th-century al-Andalus, the Arabian philosopher, Ibn Tufail (Abubacer), wrote discussions on materialism in his philosophical novel, Hayy ibn Yaqdhan (Philosophus Autodidactus), while vaguely foreshadowing the idea of a historical materialism. +The French cleric Pierre Gassendi (1592-1665) represented the materialist tradition in opposition to the attempts of René Descartes (1596-1650) to provide the natural sciences with dualist foundations. There followed the materialist and atheist abbé Jean Meslier (1664-1729), Julien Offray de La Mettrie, the German-French Paul-Henri Thiry Baron d'Holbach (1723-1789), the Encyclopedist Denis Diderot (1713-1784), and other French Enlightenment thinkers; as well as (in England) John "Walking" Stewart (1747-1822), whose insistence in seeing matter as endowed with a moral dimension had a major impact on the philosophical poetry of William Wordsworth (1770-1850). +Arthur Schopenhauer (1788-1860) wrote that "...materialism is the philosophy of the subject who forgets to take account of himself". He claimed that an observing subject can only know material objects through the mediation of the brain and its particular organization. That is, the brain itself is the "determiner" of how material objects will be experienced or perceived: +The German materialist and atheist anthropologist Ludwig Feuerbach would signal a new turn in materialism through his book, The Essence of Christianity (1841), which provided a humanist account of religion as the outward projection of man's inward nature. Feuerbach's materialism would later heavily influence Karl Marx. +Many current and recent philosophers—e.g., Daniel Dennett, Willard Van Orman Quine, Donald Davidson, and Jerry Fodor—operate within a broadly physicalist or materialist framework, producing rival accounts of how best to accommodate mind, including functionalism, anomalous monism, identity theory, and so on. +The nature and definition of matter - like other key concepts in science and philosophy - have occasioned much debate. Is there a single kind of matter (hyle) which everything is made of, or multiple kinds? Is matter a continuous substance capable of expressing multiple forms (hylomorphism), or a number of discrete, unchanging constituents (atomism)? Does it have intrinsic properties (substance theory), or is it lacking them (prima materia)? +One challenge to the traditional concept of matter as tangible "stuff" came with the rise of field physics in the 19th century. Relativity shows that matter and energy (including the spatially distributed energy of fields) are interchangeable. This enables the ontological view that energy is prima materia and matter is one of its forms. On the other hand, the Standard Model of Particle physics uses quantum field theory to describe all interactions. On this view it could be said that fields are prima materia and the energy is a property of the field. +According to the dominant cosmological model, the Lambda-CDM model, less than 5% of the universe's energy density is made up of the "matter" described by the Standard Model of Particle Physics, and the majority of the universe is composed of dark matter and dark energy - with little agreement amongst scientists about what these are made of. +With the advent of quantum physics, some scientists believed the concept of matter had merely changed, while others believed the conventional position could no longer be maintained. For instance Werner Heisenberg said "The ontology of materialism rested upon the illusion that the kind of existence, the direct 'actuality' of the world around us, can be extrapolated into the atomic range. This extrapolation, however, is impossible... atoms are not things." Likewise, some philosophers[which?] feel that these dichotomies necessitate a switch from materialism to physicalism. Others use the terms "materialism" and "physicalism" interchangeably. +Some modern day physicists and science writers—such as Paul Davies and John Gribbin—have argued that materialism has been disproven by certain scientific findings in physics, such as quantum mechanics and chaos theory. In 1991, Gribbin and Davies released their book The Matter Myth, the first chapter of which, "The Death of Materialism", contained the following passage: +Davies' and Gribbin's objections are shared by proponents of digital physics who view information rather than matter to be fundamental. Their objections were also shared by some founders of quantum theory, such as Max Planck, who wrote: +According to the Catholic Encyclopedia of 1907-1912, materialism, defined as "a philosophical system which regards matter as the only reality in the world [...] denies the existence of God and the soul". Materialism, in this view, therefore becomes incompatible with most world religions, including Christianity, Judaism, and Islam. In such a context one can conflate materialism with atheism. Most of Hinduism and transcendentalism regards all matter as an illusion called Maya, blinding humans from knowing "the truth". Maya is the limited, purely physical and mental reality in which our everyday consciousness has become entangled. Maya gets destroyed for a person when s/he perceives Brahman with transcendental knowledge. +In contrast, Joseph Smith, the founder of the Latter Day Saint movement, taught: "There is no such thing as immaterial matter. All spirit is matter, but it is more fine or pure, and can only be discerned by purer eyes; We cannot see it; but when our bodies are purified we shall see that it is all matter." This spirit element has always existed; it is co-eternal with God. It is also called "intelligence" or "the light of truth", which like all observable matter "was not created or made, neither indeed can be". Members of the Church of Jesus Christ of Latter-day Saints view the revelations of Joseph Smith as a restoration of original Christian doctrine, which they believe post-apostolic theologians began to corrupt in the centuries after Christ. The writings of many[quantify] of these theologians indicate a clear influence of Greek metaphysical philosophies such as Neoplatonism, which characterized divinity as an utterly simple, immaterial, formless, substance/essence (ousia) that transcended all that was physical. Despite strong opposition from many Christians, this metaphysical depiction of God eventually became incorporated into the doctrine of the Christian church, displacing the original Judeo-Christian concept of a physical, corporeal God who created humans in His image and likeness. +An argument for idealism, such as those of Hegel and Berkeley, is ipso facto an argument against materialism. Matter can be argued to be redundant, as in bundle theory, and mind-independent properties can in turn be reduced to subjective percepts. Berkeley presents an example of the latter by pointing out that it is impossible to gather direct evidence of matter, as there is no direct experience of matter; all that is experienced is perception, whether internal or external. As such, the existence of matter can only be assumed from the apparent (perceived) stability of perceptions; it finds absolutely no evidence in direct experience. +If matter and energy are seen as necessary to explain the physical world, but incapable of explaining mind, dualism results. Emergence, holism, and process philosophy seek to ameliorate the perceived shortcomings of traditional (especially mechanistic) materialism without abandoning materialism entirely. +Some critics object to materialism as part of an overly skeptical, narrow or reductivist approach to theorizing, rather than to the ontological claim that matter is the only substance. Particle physicist and Anglican theologian John Polkinghorne objects to what he calls promissory materialism — claims that materialistic science will eventually succeed in explaining phenomena it has not so far been able to explain. Polkinghorne prefers "dual-aspect monism" to faith in materialism. diff --git a/e2e/samples/test_documents/Normans.txt b/e2e/samples/test_documents/Normans.txt new file mode 100644 index 0000000000..d8f2f31e7b --- /dev/null +++ b/e2e/samples/test_documents/Normans.txt @@ -0,0 +1,45 @@ +The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse ("Norman" comes from "Norseman") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries. +The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands. +The English name "Normans" comes from the French words Normans/Normanz, plural of Normant, modern French normand, which is itself borrowed from Old Low Franconian Nortmann "Northman" or directly from Old Norse Norðmaðr, Latinized variously as Nortmannus, Normannus, or Nordmannus (recorded in Medieval Latin, 9th century) to mean "Norseman, Viking". +In the course of the 10th century, the initially destructive incursions of Norse war bands into the rivers of France evolved into more permanent encampments that included local women and personal property. The Duchy of Normandy, which began in 911 as a fiefdom, was established by the treaty of Saint-Clair-sur-Epte between King Charles III of West Francia and the famed Viking ruler Rollo, and was situated in the former Frankish kingdom of Neustria. The treaty offered Rollo and his men the French lands between the river Epte and the Atlantic coast in exchange for their protection against further Viking incursions. The area corresponded to the northern part of present-day Upper Normandy down to the river Seine, but the Duchy would eventually extend west beyond the Seine. The territory was roughly equivalent to the old province of Rouen, and reproduced the Roman administrative structure of Gallia Lugdunensis II (part of the former Gallia Lugdunensis). +Before Rollo's arrival, its populations did not differ from Picardy or the Île-de-France, which were considered "Frankish". Earlier Viking settlers had begun arriving in the 880s, but were divided between colonies in the east (Roumois and Pays de Caux) around the low Seine valley and in the west in the Cotentin Peninsula, and were separated by traditional pagii, where the population remained about the same with almost no foreign settlers. Rollo's contingents who raided and ultimately settled Normandy and parts of the Atlantic coast included Danes, Norwegians, Norse–Gaels, Orkney Vikings, possibly Swedes, and Anglo-Danes from the English Danelaw under Norse control. +The descendants of Rollo's Vikings and their Frankish wives would replace the Norse religion and Old Norse language with Catholicism (Christianity) and the Gallo-Romance language of the local people, blending their maternal Frankish heritage with Old Norse traditions and customs to synthesize a unique "Norman" culture in the north of France. The Norman language was forged by the adoption of the indigenous langue d'oïl branch of Romance by a Norse-speaking ruling class, and it developed into the regional language that survives today. +The Normans thereafter adopted the growing feudal doctrines of the rest of France, and worked them into a functional hierarchical system in both Normandy and in England. The new Norman rulers were culturally and ethnically distinct from the old French aristocracy, most of whom traced their lineage to Franks of the Carolingian dynasty. Most Norman knights remained poor and land-hungry, and by 1066 Normandy had been exporting fighting horsemen for more than a generation. Many Normans of Italy, France and England eventually served as avid Crusaders under the Italo-Norman prince Bohemund I and the Anglo-Norman king Richard the Lion-Heart. +Opportunistic bands of Normans successfully established a foothold in Southern Italy (the Mezzogiorno). Probably as the result of returning pilgrims' stories, the Normans entered the Mezzogiorno as warriors in 1017 at the latest. In 999, according to Amatus of Montecassino, Norman pilgrims returning from Jerusalem called in at the port of Salerno when a Saracen attack occurred. The Normans fought so valiantly that Prince Guaimar III begged them to stay, but they refused and instead offered to tell others back home of the prince's request. William of Apulia tells that, in 1016, Norman pilgrims to the shrine of the Archangel Michael at Monte Gargano were met by Melus of Bari, a Lombard nobleman and rebel, who persuaded them to return with more warriors to help throw off the Byzantine rule, which they did. +The two most prominent Norman families to arrive in the Mediterranean were descendants of Tancred of Hauteville and the Drengot family, of whom Rainulf Drengot received the county of Aversa, the first Norman toehold in the south, from Duke Sergius IV of Naples in 1030. The Hauteville family achieved princely rank by proclaiming prince Guaimar IV of Salerno "Duke of Apulia and Calabria". He promptly awarded their elected leader, William Iron Arm, with the title of count in his capital of Melfi. The Drengot family thereafter attained the principality of Capua, and emperor Henry III legally ennobled the Hauteville leader, Drogo, as "dux et magister Italiae comesque Normannorum totius Apuliae et Calabriae" ("Duke and Master of Italy and Count of the Normans of all Apulia and Calabria") in 1047. +From these bases, the Normans eventually captured Sicily and Malta from the Saracens, under the leadership of the famous Robert Guiscard, a Hauteville, and his younger brother Roger the Great Count. Roger's son, Roger II of Sicily, was crowned king in 1130 (exactly one century after Rainulf was "crowned" count) by Antipope Anacletus II. The Kingdom of Sicily lasted until 1194, when it was transferred to the House of Hohenstaufen through marriage. The Normans left their legacy in many castles, such as William Iron Arm's citadel at Squillace, and cathedrals, such as Roger II's Cappella Palatina chapel at Palermo, which dot the landscape and give a wholly distinct architectural flavor to accompany its unique history. +Institutionally, the Normans combined the administrative machinery of the Byzantines, Arabs, and Lombards with their own conceptions of feudal law and order to forge a unique government. Under this state, there was great religious freedom, and alongside the Norman nobles existed a meritocratic bureaucracy of Jews, Muslims and Christians, both Catholic and Eastern Orthodox. The Kingdom of Sicily thus became characterized by Norman, Byzantine Greek, Arab, Lombard and "native" Sicilian populations living in harmony, and its Norman rulers fostered plans of establishing an Empire that would have encompassed Fatimid Egypt as well as the Crusader states in the Levant. One of the great geographical treatises of the Middle Ages, the "Tabula Rogeriana", was written by the Andalusian al-Idrisi for king Roger II of Sicily, and entitled "Kitab Rudjdjar" ("The Book of Roger"). +Soon after the Normans began to enter Italy, they entered the Byzantine Empire and then Armenia, fighting against the Pechenegs, the Bulgars, and especially the Seljuk Turks. Norman mercenaries were first encouraged to come to the south by the Lombards to act against the Byzantines, but they soon fought in Byzantine service in Sicily. They were prominent alongside Varangian and Lombard contingents in the Sicilian campaign of George Maniaces in 1038–40. There is debate whether the Normans in Greek service actually were from Norman Italy, and it now seems likely only a few came from there. It is also unknown how many of the "Franks", as the Byzantines called them, were Normans and not other Frenchmen. +One of the first Norman mercenaries to serve as a Byzantine general was Hervé in the 1050s. By then however, there were already Norman mercenaries serving as far away as Trebizond and Georgia. They were based at Malatya and Edessa, under the Byzantine duke of Antioch, Isaac Komnenos. In the 1060s, Robert Crispin led the Normans of Edessa against the Turks. Roussel de Bailleul even tried to carve out an independent state in Asia Minor with support from the local population, but he was stopped by the Byzantine general Alexius Komnenos. +Some Normans joined Turkish forces to aid in the destruction of the Armenians vassal-states of Sassoun and Taron in far eastern Anatolia. Later, many took up service with the Armenian state further south in Cilicia and the Taurus Mountains. A Norman named Oursel led a force of "Franks" into the upper Euphrates valley in northern Syria. From 1073 to 1074, 8,000 of the 20,000 troops of the Armenian general Philaretus Brachamius were Normans—formerly of Oursel—led by Raimbaud. They even lent their ethnicity to the name of their castle: Afranji, meaning "Franks." The known trade between Amalfi and Antioch and between Bari and Tarsus may be related to the presence of Italo-Normans in those cities while Amalfi and Bari were under Norman rule in Italy. +Several families of Byzantine Greece were of Norman mercenary origin during the period of the Comnenian Restoration, when Byzantine emperors were seeking out western European warriors. The Raoulii were descended from an Italo-Norman named Raoul, the Petraliphae were descended from a Pierre d'Aulps, and that group of Albanian clans known as the Maniakates were descended from Normans who served under George Maniaces in the Sicilian expedition of 1038. +Robert Guiscard, an other Norman adventurer previously elevated to the dignity of count of Apulia as the result of his military successes, ultimately drove the Byzantines out of southern Italy. Having obtained the consent of pope Gregory VII and acting as his vassal, Robert continued his campaign conquering the Balkan peninsula as a foothold for western feudal lords and the Catholic Church. After allying himself with Croatia and the Catholic cities of Dalmatia, in 1081 he led an army of 30,000 men in 300 ships landing on the southern shores of Albania, capturing Valona, Kanina, Jericho (Orikumi), and reaching Butrint after numerous pillages. They joined the fleet that had previously conquered Corfu and attacked Dyrrachium from land and sea, devastating everything along the way. Under these harsh circumstances, the locals accepted the call of emperor Alexius I Comnenus to join forces with the Byzantines against the Normans. The Albanian forces could not take part in the ensuing battle because it had started before their arrival. Immediately before the battle, the Venetian fleet had secured a victory in the coast surrounding the city. Forced to retreat, Alexius ceded the command to a high Albanian official named Comiscortes in the service of Byzantium. The city's garrison resisted until February 1082, when Dyrrachium was betrayed to the Normans by the Venetian and Amalfitan merchants who had settled there. The Normans were now free to penetrate into the hinterland; they took Ioannina and some minor cities in southwestern Macedonia and Thessaly before appearing at the gates of Thessalonica. Dissension among the high ranks coerced the Normans to retreat to Italy. They lost Dyrrachium, Valona, and Butrint in 1085, after the death of Robert. +A few years after the First Crusade, in 1107, the Normans under the command of Bohemond, Robert's son, landed in Valona and besieged Dyrrachium using the most sophisticated military equipment of the time, but to no avail. Meanwhile, they occupied Petrela, the citadel of Mili at the banks of the river Deabolis, Gllavenica (Ballsh), Kanina and Jericho. This time, the Albanians sided with the Normans, dissatisfied by the heavy taxes the Byzantines had imposed upon them. With their help, the Normans secured the Arbanon passes and opened their way to Dibra. The lack of supplies, disease and Byzantine resistance forced Bohemond to retreat from his campaign and sign a peace treaty with the Byzantines in the city of Deabolis. +The further decline of Byzantine state-of-affairs paved the road to a third attack in 1185, when a large Norman army invaded Dyrrachium, owing to the betrayal of high Byzantine officials. Some time later, Dyrrachium—one of the most important naval bases of the Adriatic—fell again to Byzantine hands. +The Normans were in contact with England from an early date. Not only were their original Viking brethren still ravaging the English coasts, they occupied most of the important ports opposite England across the English Channel. This relationship eventually produced closer ties of blood through the marriage of Emma, sister of Duke Richard II of Normandy, and King Ethelred II of England. Because of this, Ethelred fled to Normandy in 1013, when he was forced from his kingdom by Sweyn Forkbeard. His stay in Normandy (until 1016) influenced him and his sons by Emma, who stayed in Normandy after Cnut the Great's conquest of the isle. +When finally Edward the Confessor returned from his father's refuge in 1041, at the invitation of his half-brother Harthacnut, he brought with him a Norman-educated mind. He also brought many Norman counsellors and fighters, some of whom established an English cavalry force. This concept never really took root, but it is a typical example of the attitudes of Edward. He appointed Robert of Jumièges archbishop of Canterbury and made Ralph the Timid earl of Hereford. He invited his brother-in-law Eustace II, Count of Boulogne to his court in 1051, an event which resulted in the greatest of early conflicts between Saxon and Norman and ultimately resulted in the exile of Earl Godwin of Wessex. +In 1066, Duke William II of Normandy conquered England killing King Harold II at the Battle of Hastings. The invading Normans and their descendants replaced the Anglo-Saxons as the ruling class of England. The nobility of England were part of a single Normans culture and many had lands on both sides of the channel. Early Norman kings of England, as Dukes of Normandy, owed homage to the King of France for their land on the continent. They considered England to be their most important holding (it brought with it the title of King—an important status symbol). +Eventually, the Normans merged with the natives, combining languages and traditions. In the course of the Hundred Years' War, the Norman aristocracy often identified themselves as English. The Anglo-Norman language became distinct from the Latin language, something that was the subject of some humour by Geoffrey Chaucer. The Anglo-Norman language was eventually absorbed into the Anglo-Saxon language of their subjects (see Old English) and influenced it, helping (along with the Norse language of the earlier Anglo-Norse settlers and the Latin used by the church) in the development of Middle English. It in turn evolved into Modern English. +The Normans had a profound effect on Irish culture and history after their invasion at Bannow Bay in 1169. Initially the Normans maintained a distinct culture and ethnicity. Yet, with time, they came to be subsumed into Irish culture to the point that it has been said that they became "more Irish than the Irish themselves." The Normans settled mostly in an area in the east of Ireland, later known as the Pale, and also built many fine castles and settlements, including Trim Castle and Dublin Castle. Both cultures intermixed, borrowing from each other's language, culture and outlook. Norman descendants today can be recognised by their surnames. Names such as French, (De) Roche, Devereux, D'Arcy, Treacy and Lacy are particularly common in the southeast of Ireland, especially in the southern part of County Wexford where the first Norman settlements were established. Other Norman names such as Furlong predominate there. Another common Norman-Irish name was Morell (Murrell) derived from the French Norman name Morel. Other names beginning with Fitz (from the Norman for son) indicate Norman ancestry. These included Fitzgerald, FitzGibbons (Gibbons) dynasty, Fitzmaurice. Other families bearing such surnames as Barry (de Barra) and De Búrca (Burke) are also of Norman extraction. +One of the claimants of the English throne opposing William the Conqueror, Edgar Atheling, eventually fled to Scotland. King Malcolm III of Scotland married Edgar's sister Margaret, and came into opposition to William who had already disputed Scotland's southern borders. William invaded Scotland in 1072, riding as far as Abernethy where he met up with his fleet of ships. Malcolm submitted, paid homage to William and surrendered his son Duncan as a hostage, beginning a series of arguments as to whether the Scottish Crown owed allegiance to the King of England. +Normans came into Scotland, building castles and founding noble families who would provide some future kings, such as Robert the Bruce, as well as founding a considerable number of the Scottish clans. King David I of Scotland, whose elder brother Alexander I had married Sybilla of Normandy, was instrumental in introducing Normans and Norman culture to Scotland, part of the process some scholars call the "Davidian Revolution". Having spent time at the court of Henry I of England (married to David's sister Maud of Scotland), and needing them to wrestle the kingdom from his half-brother Máel Coluim mac Alaxandair, David had to reward many with lands. The process was continued under David's successors, most intensely of all under William the Lion. The Norman-derived feudal system was applied in varying degrees to most of Scotland. Scottish families of the names Bruce, Gray, Ramsay, Fraser, Ogilvie, Montgomery, Sinclair, Pollock, Burnard, Douglas and Gordon to name but a few, and including the later royal House of Stewart, can all be traced back to Norman ancestry. +Even before the Norman Conquest of England, the Normans had come into contact with Wales. Edward the Confessor had set up the aforementioned Ralph as earl of Hereford and charged him with defending the Marches and warring with the Welsh. In these original ventures, the Normans failed to make any headway into Wales. +Subsequent to the Conquest, however, the Marches came completely under the dominance of William's most trusted Norman barons, including Bernard de Neufmarché, Roger of Montgomery in Shropshire and Hugh Lupus in Cheshire. These Normans began a long period of slow conquest during which almost all of Wales was at some point subject to Norman interference. Norman words, such as baron (barwn), first entered Welsh at that time. +The legendary religious zeal of the Normans was exercised in religious wars long before the First Crusade carved out a Norman principality in Antioch. They were major foreign participants in the Reconquista in Iberia. In 1018, Roger de Tosny travelled to the Iberian Peninsula to carve out a state for himself from Moorish lands, but failed. In 1064, during the War of Barbastro, William of Montreuil led the papal army and took a huge booty. +In 1096, Crusaders passing by the siege of Amalfi were joined by Bohemond of Taranto and his nephew Tancred with an army of Italo-Normans. Bohemond was the de facto leader of the Crusade during its passage through Asia Minor. After the successful Siege of Antioch in 1097, Bohemond began carving out an independent principality around that city. Tancred was instrumental in the conquest of Jerusalem and he worked for the expansion of the Crusader kingdom in Transjordan and the region of Galilee.[citation needed] +The conquest of Cyprus by the Anglo-Norman forces of the Third Crusade opened a new chapter in the history of the island, which would be under Western European domination for the following 380 years. Although not part of a planned operation, the conquest had much more permanent results than initially expected. +In April 1191 Richard the Lion-hearted left Messina with a large fleet in order to reach Acre. But a storm dispersed the fleet. After some searching, it was discovered that the boat carrying his sister and his fiancée Berengaria was anchored on the south coast of Cyprus, together with the wrecks of several other ships, including the treasure ship. Survivors of the wrecks had been taken prisoner by the island's despot Isaac Komnenos. On 1 May 1191, Richard's fleet arrived in the port of Limassol on Cyprus. He ordered Isaac to release the prisoners and the treasure. Isaac refused, so Richard landed his troops and took Limassol. +Various princes of the Holy Land arrived in Limassol at the same time, in particular Guy de Lusignan. All declared their support for Richard provided that he support Guy against his rival Conrad of Montferrat. The local barons abandoned Isaac, who considered making peace with Richard, joining him on the crusade, and offering his daughter in marriage to the person named by Richard. But Isaac changed his mind and tried to escape. Richard then proceeded to conquer the whole island, his troops being led by Guy de Lusignan. Isaac surrendered and was confined with silver chains, because Richard had promised that he would not place him in irons. By 1 June, Richard had conquered the whole island. His exploit was well publicized and contributed to his reputation; he also derived significant financial gains from the conquest of the island. Richard left for Acre on 5 June, with his allies. Before his departure, he named two of his Norman generals, Richard de Camville and Robert de Thornham, as governors of Cyprus. +While in Limassol, Richard the Lion-Heart married Berengaria of Navarre, first-born daughter of King Sancho VI of Navarre. The wedding was held on 12 May 1191 at the Chapel of St. George and it was attended by Richard's sister Joan, whom he had brought from Sicily. The marriage was celebrated with great pomp and splendor. Among other grand ceremonies was a double coronation: Richard caused himself to be crowned King of Cyprus, and Berengaria Queen of England and Queen of Cyprus as well. +The rapid Anglo-Norman conquest proved more important than it seemed. The island occupied a key strategic position on the maritime lanes to the Holy Land, whose occupation by the Christians could not continue without support from the sea. Shortly after the conquest, Cyprus was sold to the Knights Templar and it was subsequently acquired, in 1192, by Guy de Lusignan and became a stable feudal kingdom. It was only in 1489 that the Venetians acquired full control of the island, which remained a Christian stronghold until the fall of Famagusta in 1571. +Between 1402 and 1405, the expedition led by the Norman noble Jean de Bethencourt and the Poitevine Gadifer de la Salle conquered the Canarian islands of Lanzarote, Fuerteventura and El Hierro off the Atlantic coast of Africa. Their troops were gathered in Normandy, Gascony and were later reinforced by Castilian colonists. +Bethencourt took the title of King of the Canary Islands, as vassal to Henry III of Castile. In 1418, Jean's nephew Maciot de Bethencourt sold the rights to the islands to Enrique Pérez de Guzmán, 2nd Count de Niebla. +The customary law of Normandy was developed between the 10th and 13th centuries and survives today through the legal systems of Jersey and Guernsey in the Channel Islands. Norman customary law was transcribed in two customaries in Latin by two judges for use by them and their colleagues: These are the Très ancien coutumier (Very ancient customary), authored between 1200 and 1245; and the Grand coutumier de Normandie (Great customary of Normandy, originally Summa de legibus Normanniae in curia laïcali), authored between 1235 and 1245. +Norman architecture typically stands out as a new stage in the architectural history of the regions they subdued. They spread a unique Romanesque idiom to England and Italy, and the encastellation of these regions with keeps in their north French style fundamentally altered the military landscape. Their style was characterised by rounded arches, particularly over windows and doorways, and massive proportions. +In England, the period of Norman architecture immediately succeeds that of the Anglo-Saxon and precedes the Early Gothic. In southern Italy, the Normans incorporated elements of Islamic, Lombard, and Byzantine building techniques into their own, initiating a unique style known as Norman-Arab architecture within the Kingdom of Sicily. +In the visual arts, the Normans did not have the rich and distinctive traditions of the cultures they conquered. However, in the early 11th century the dukes began a programme of church reform, encouraging the Cluniac reform of monasteries and patronising intellectual pursuits, especially the proliferation of scriptoria and the reconstitution of a compilation of lost illuminated manuscripts. The church was utilised by the dukes as a unifying force for their disparate duchy. The chief monasteries taking part in this "renaissance" of Norman art and scholarship were Mont-Saint-Michel, Fécamp, Jumièges, Bec, Saint-Ouen, Saint-Evroul, and Saint-Wandrille. These centres were in contact with the so-called "Winchester school", which channeled a pure Carolingian artistic tradition to Normandy. In the final decade of the 11th and first of the 12th century, Normandy experienced a golden age of illustrated manuscripts, but it was brief and the major scriptoria of Normandy ceased to function after the midpoint of the century. +The French Wars of Religion in the 16th century and French Revolution in the 18th successively destroyed much of what existed in the way of the architectural and artistic remnant of this Norman creativity. The former, with their violence, caused the wanton destruction of many Norman edifices; the latter, with its assault on religion, caused the purposeful destruction of religious objects of any type, and its destabilisation of society resulted in rampant pillaging. +By far the most famous work of Norman art is the Bayeux Tapestry, which is not a tapestry but a work of embroidery. It was commissioned by Odo, the Bishop of Bayeux and first Earl of Kent, employing natives from Kent who were learned in the Nordic traditions imported in the previous half century by the Danish Vikings. +In Britain, Norman art primarily survives as stonework or metalwork, such as capitals and baptismal fonts. In southern Italy, however, Norman artwork survives plentifully in forms strongly influenced by its Greek, Lombard, and Arab forebears. Of the royal regalia preserved in Palermo, the crown is Byzantine in style and the coronation cloak is of Arab craftsmanship with Arabic inscriptions. Many churches preserve sculptured fonts, capitals, and more importantly mosaics, which were common in Norman Italy and drew heavily on the Greek heritage. Lombard Salerno was a centre of ivorywork in the 11th century and this continued under Norman domination. Finally should be noted the intercourse between French Crusaders traveling to the Holy Land who brought with them French artefacts with which to gift the churches at which they stopped in southern Italy amongst their Norman cousins. For this reason many south Italian churches preserve works from France alongside their native pieces. +Normandy was the site of several important developments in the history of classical music in the 11th century. Fécamp Abbey and Saint-Evroul Abbey were centres of musical production and education. At Fécamp, under two Italian abbots, William of Volpiano and John of Ravenna, the system of denoting notes by letters was developed and taught. It is still the most common form of pitch representation in English- and German-speaking countries today. Also at Fécamp, the staff, around which neumes were oriented, was first developed and taught in the 11th century. Under the German abbot Isembard, La Trinité-du-Mont became a centre of musical composition. +At Saint Evroul, a tradition of singing had developed and the choir achieved fame in Normandy. Under the Norman abbot Robert de Grantmesnil, several monks of Saint-Evroul fled to southern Italy, where they were patronised by Robert Guiscard and established a Latin monastery at Sant'Eufemia. There they continued the tradition of singing. diff --git a/e2e/samples/test_documents/Strasbourg.txt b/e2e/samples/test_documents/Strasbourg.txt new file mode 100644 index 0000000000..bf64384234 --- /dev/null +++ b/e2e/samples/test_documents/Strasbourg.txt @@ -0,0 +1,34 @@ +Strasbourg (/ˈstræzbɜːrɡ/, French pronunciation: ​[stʁaz.buʁ, stʁas.buʁ]; Alsatian: Strossburi; German: Straßburg, [ˈʃtʁaːsbʊɐ̯k]) is the capital and largest city of the Alsace-Champagne-Ardenne-Lorraine (ACAL) region in eastern France and is the official seat of the European Parliament. Located close to the border with Germany, it is the capital of the Bas-Rhin département. The city and the region of Alsace were historically predominantly Alemannic-speaking, hence the city's Germanic name. In 2013, the city proper had 275,718 inhabitants, Eurométropole de Strasbourg (Greater Strasbourg) had 475,934 inhabitants and the Arrondissement of Strasbourg had 482,384 inhabitants. With a population of 768,868 in 2012, Strasbourg's metropolitan area (only the part of the metropolitan area on French territory) is the ninth largest in France and home to 13% of the ACAL region's inhabitants. The transnational Eurodistrict Strasbourg-Ortenau had a population of 915,000 inhabitants in 2014. +Strasbourg's historic city centre, the Grande Île (Grand Island), was classified a World Heritage site by UNESCO in 1988, the first time such an honour was placed on an entire city centre. Strasbourg is immersed in the Franco-German culture and although violently disputed throughout history, has been a bridge of unity between France and Germany for centuries, especially through the University of Strasbourg, currently the second largest in France, and the coexistence of Catholic and Protestant culture. The largest Islamic place of worship in France, the Strasbourg Grand Mosque, was inaugurated by French Interior Minister Manuel Valls on 27 September 2012. +Strasbourg is situated on the eastern border of France with Germany. This border is formed by the River Rhine, which also forms the eastern border of the modern city, facing across the river to the German town Kehl. The historic core of Strasbourg however lies on the Grande Île in the River Ill, which here flows parallel to, and roughly 4 kilometres (2.5 mi) from, the Rhine. The natural courses of the two rivers eventually join some distance downstream of Strasbourg, although several artificial waterways now connect them within the city. +The Romans under Nero Claudius Drusus established a military outpost belonging to the Germania Superior Roman province at Strasbourg's current location, and named it Argentoratum. (Hence the town is commonly called Argentina in medieval Latin.) The name "Argentoratum" was first mentioned in 12 BC and the city celebrated its 2,000th birthday in 1988. "Argentorate" as the toponym of the Gaulish settlement preceded it before being Latinized, but it is not known by how long. The Roman camp was destroyed by fire and rebuilt six times between the first and the fifth centuries AD: in 70, 97, 235, 355, in the last quarter of the fourth century, and in the early years of the fifth century. It was under Trajan and after the fire of 97 that Argentoratum received its most extended and fortified shape. From the year 90 on, the Legio VIII Augusta was permanently stationed in the Roman camp of Argentoratum. It then included a cavalry section and covered an area of approximately 20 hectares. Other Roman legions temporarily stationed in Argentoratum were the Legio XIV Gemina and the Legio XXI Rapax, the latter during the reign of Nero. +The centre of Argentoratum proper was situated on the Grande Île (Cardo: current Rue du Dôme, Decumanus: current Rue des Hallebardes). The outline of the Roman "castrum" is visible in the street pattern in the Grande Ile. Many Roman artifacts have also been found along the current Route des Romains, the road that led to Argentoratum, in the suburb of Kœnigshoffen. This was where the largest burial places were situated, as well as the densest concentration of civilian dwelling places and commerces next to the camp. Among the most outstanding finds in Kœnigshoffen were (found in 1911–12) the fragments of a grand Mithraeum that had been shattered by early Christians in the fourth century. From the fourth century, Strasbourg was the seat of the Bishopric of Strasbourg (made an Archbishopric in 1988). Archaeological excavations below the current Église Saint-Étienne in 1948 and 1956 unearthed the apse of a church dating back to the late fourth or early fifth century, considered to be the oldest church in Alsace. It is supposed that this was the first seat of the Roman Catholic Diocese of Strasbourg. +In the fifth century Strasbourg was occupied successively by Alemanni, Huns, and Franks. In the ninth century it was commonly known as Strazburg in the local language, as documented in 842 by the Oaths of Strasbourg. This trilingual text contains, alongside texts in Latin and Old High German (teudisca lingua), the oldest written variety of Gallo-Romance (lingua romana) clearly distinct from Latin, the ancestor of Old French. The town was also called Stratisburgum or Strateburgus in Latin, from which later came Strossburi in Alsatian and Straßburg in Standard German, and then Strasbourg in French. The Oaths of Strasbourg is considered as marking the birth of the two countries of France and Germany with the division of the Carolingian Empire. +A revolution in 1332 resulted in a broad-based city government with participation of the guilds, and Strasbourg declared itself a free republic. The deadly bubonic plague of 1348 was followed on 14 February 1349 by one of the first and worst pogroms in pre-modern history: over a thousand Jews were publicly burnt to death, with the remainder of the Jewish population being expelled from the city. Until the end of the 18th century, Jews were forbidden to remain in town after 10 pm. The time to leave the city was signalled by a municipal herald blowing the Grüselhorn (see below, Museums, Musée historique);. A special tax, the Pflastergeld (pavement money), was furthermore to be paid for any horse that a Jew would ride or bring into the city while allowed to. +In the 1520s during the Protestant Reformation, the city, under the political guidance of Jacob Sturm von Sturmeck and the spiritual guidance of Martin Bucer embraced the religious teachings of Martin Luther. Their adherents established a Gymnasium, headed by Johannes Sturm, made into a University in the following century. The city first followed the Tetrapolitan Confession, and then the Augsburg Confession. Protestant iconoclasm caused much destruction to churches and cloisters, notwithstanding that Luther himself opposed such a practice. Strasbourg was a centre of humanist scholarship and early book-printing in the Holy Roman Empire, and its intellectual and political influence contributed much to the establishment of Protestantism as an accepted denomination in the southwest of Germany. (John Calvin spent several years as a political refugee in the city). The Strasbourg Councillor Sturm and guildmaster Matthias represented the city at the Imperial Diet of Speyer (1529), where their protest led to the schism of the Catholic Church and the evolution of Protestantism. Together with four other free cities, Strasbourg presented the confessio tetrapolitana as its Protestant book of faith at the Imperial Diet of Augsburg in 1530, where the slightly different Augsburg Confession was also handed over to Charles V, Holy Roman Emperor. +Louis' advisors believed that, as long as Strasbourg remained independent, it would endanger the King's newly annexed territories in Alsace, and, that to defend these large rural lands effectively, a garrison had to be placed in towns such as Strasbourg. Indeed, the bridge over the Rhine at Strasbourg had been used repeatedly by Imperial (Holy Roman Empire) forces, and three times during the Franco-Dutch War Strasbourg had served as a gateway for Imperial invasions into Alsace. In September 1681 Louis' forces, though lacking a clear casus belli, surrounded the city with overwhelming force. After some negotiation, Louis marched into the city unopposed on 30 September 1681 and proclaimed its annexation. +This annexation was one of the direct causes of the brief and bloody War of the Reunions whose outcome left the French in possession. The French annexation was recognized by the Treaty of Ryswick (1697). The official policy of religious intolerance which drove most Protestants from France after the revocation of the Edict of Nantes in 1685 was not applied in Strasbourg and in Alsace, because both had a special status as a province à l'instar de l'étranger effectif (a kind of foreign province of the king of France). Strasbourg Cathedral, however, was taken from the Lutherans to be returned to the Catholics as the French authorities tried to promote Catholicism wherever they could (some other historic churches remained in Protestant hands). Its language also remained overwhelmingly German: the German Lutheran university persisted until the French Revolution. Famous students included Goethe and Herder. +Strasbourg's status as a free city was revoked by the French Revolution. Enragés, most notoriously Eulogius Schneider, ruled the city with an increasingly iron hand. During this time, many churches and monasteries were either destroyed or severely damaged. The cathedral lost hundreds of its statues (later replaced by copies in the 19th century) and in April 1794, there was talk of tearing its spire down, on the grounds that it was against the principle of equality. The tower was saved, however, when in May of the same year citizens of Strasbourg crowned it with a giant tin Phrygian cap. This artifact was later kept in the historical collections of the city until it was destroyed by the Germans in 1870 during the Franco-Prussian war. +During the Franco-Prussian War and the Siege of Strasbourg, the city was heavily bombarded by the Prussian army. The bombardment of the city was meant to break the morale of the people of Strasbourg. On 24 and 26 August 1870, the Museum of Fine Arts was destroyed by fire, as was the Municipal Library housed in the Gothic former Dominican church, with its unique collection of medieval manuscripts (most famously the Hortus deliciarum), rare Renaissance books, archeological finds and historical artifacts. The gothic cathedral was damaged as well as the medieval church of Temple Neuf, the theatre, the city hall, the court of justice and many houses. At the end of the siege 10,000 inhabitants were left without shelter; over 600 died, including 261 civilians, and 3200 were injured, including 1,100 civilians. +In 1871, after the end of the war, the city was annexed to the newly established German Empire as part of the Reichsland Elsass-Lothringen under the terms of the Treaty of Frankfurt. As part of Imperial Germany, Strasbourg was rebuilt and developed on a grand and representative scale, such as the Neue Stadt, or "new city" around the present Place de la République. Historian Rodolphe Reuss and Art historian Wilhelm von Bode were in charge of rebuilding the municipal archives, libraries and museums. The University, founded in 1567 and suppressed during the French Revolution as a stronghold of German sentiment,[citation needed] was reopened in 1872 under the name Kaiser-Wilhelms-Universität. +A belt of massive fortifications was established around the city, most of which still stands today, renamed after French generals and generally classified as Monuments historiques; most notably Fort Roon (now Fort Desaix) and Fort Podbielski (now Fort Ducrot) in Mundolsheim, Fort von Moltke (now Fort Rapp) in Reichstett, Fort Bismarck (now Fort Kléber) in Wolfisheim, Fort Kronprinz (now Fort Foch) in Niederhausbergen, Fort Kronprinz von Sachsen (now Fort Joffre) in Holtzheim and Fort Großherzog von Baden (now Fort Frère) in Oberhausbergen. +Following the defeat of the German empire in World War I and the abdication of the German Emperor, some revolutionary insurgents declared Alsace-Lorraine as an independent Republic, without preliminary referendum or vote. On 11 November 1918 (Armistice Day), communist insurgents proclaimed a "soviet government" in Strasbourg, following the example of Kurt Eisner in Munich as well as other German towns. French troops commanded by French general Henri Gouraud entered triumphantly in the city on 22 November. A major street of the city now bears the name of that date (Rue du 22 Novembre) which celebrates the entry of the French in the city. Viewing the massive cheering crowd gathered under the balcony of Strasbourg's town hall, French President Raymond Poincaré stated that "the plebiscite is done". +In 1919, following the Treaty of Versailles, the city was restituted to France in accordance with U.S. President Woodrow Wilson's "Fourteen Points" without a referendum. The date of the assignment was retroactively established on Armistice Day. It is doubtful whether a referendum in Strasbourg would have ended in France's favour since the political parties striving for an autonomous Alsace or a connection to France accounted only for a small proportion of votes in the last Reichstag as well as in the local elections. The Alsatian autonomists who were pro French had won many votes in the more rural parts of the region and other towns since the annexation of the region by Germany in 1871. The movement started with the first election for the Reichstag; those elected were called "les députés protestataires", and until the fall of Bismarck in 1890, they were the only deputies elected by the Alsatians to the German parliament demanding the return of those territories to France. At the last Reichstag election in Strasbourg and its periphery, the clear winners were the Social Democrats; the city was the administrative capital of the region, was inhabited by many Germans appointed by the central government in Berlin and its flourishing economy attracted many Germans. This could explain the difference between the rural vote and the one in Strasbourg. After the war, many Germans left Strasbourg and went back to Germany; some of them were denounced by the locals or expelled by the newly appointed authorities. The Saverne Affair was vivid in the memory among the Alsatians. +Between the German invasion of Poland on 1 September 1939 and the Anglo-French declaration of War against the German Reich on 3 September 1939, the entire city (a total of 120,000 people) was evacuated, like other border towns as well. Until the arrival of the Wehrmacht troops mid-June 1940, the city was, for ten months, completely empty, with the exception of the garrisoned soldiers. The Jews of Strasbourg had been evacuated to Périgueux and Limoges, the University had been evacuated to Clermont-Ferrand. +After the ceasefire following the Fall of France in June 1940, Alsace was annexed to Germany and a rigorous policy of Germanisation was imposed upon it by the Gauleiter Robert Heinrich Wagner. When, in July 1940, the first evacuees were allowed to return, only residents of Alsatian origin were admitted. The last Jews were deported on 15 July 1940 and the main synagogue, a huge Romanesque revival building that had been a major architectural landmark with its 54-metre-high dome since its completion in 1897, was set ablaze, then razed. +In September 1940 the first Alsatian resistance movement led by Marcel Weinum called La main noire (The black hand) was created. It was composed by a group of 25 young men aged from 14 to 18 years old who led several attacks against the German occupation. The actions culminated with the attack of the Gauleiter Robert Wagner, the highest commander of Alsace directly under the order of Hitler. In March 1942, Marcel Weinum was prosecuted by the Gestapo and sentenced to be beheaded at the age of 18 in April 1942 in Stuttgart, Germany. His last words will be: "If I have to die, I shall die but with a pure heart". From 1943 the city was bombarded by Allied aircraft. While the First World War had not notably damaged the city, Anglo-American bombing caused extensive destruction in raids of which at least one was allegedly carried out by mistake. In August 1944, several buildings in the Old Town were damaged by bombs, particularly the Palais Rohan, the Old Customs House (Ancienne Douane) and the Cathedral. On 23 November 1944, the city was officially liberated by the 2nd French Armoured Division under General Leclerc. He achieved the oath that he made with his soldiers, after the decisive Capture of Kufra. With the Oath of Kuffra, they swore to keep up the fight until the French flag flew over the Cathedral of Strasbourg. +Many people from Strasbourg were incorporated in the German Army against their will, and were sent to the eastern front, those young men and women were called Malgré-nous. Many tried to escape from the incorporation, join the French Resistance, or desert the Wehrmacht but many couldn't because they were running the risk of having their families sent to work or concentration camps by the Germans. Many of these men, especially those who did not answer the call immediately, were pressured to "volunteer" for service with the SS, often by direct threats on their families. This threat obliged the majority of them to remain in the German army. After the war, the few that survived were often accused of being traitors or collaborationists, because this tough situation was not known in the rest of France, and they had to face the incomprehension of many. In July 1944, 1500 malgré-nous were released from Soviet captivity and sent to Algiers, where they joined the Free French Forces. Nowadays history recognizes the suffering of those people, and museums, public discussions and memorials have been built to commemorate this terrible period of history of this part of Eastern France (Alsace and Moselle). Liberation of Strasbourg took place on 23 November 1944. +In 1949, the city was chosen to be the seat of the Council of Europe with its European Court of Human Rights and European Pharmacopoeia. Since 1952, the European Parliament has met in Strasbourg, which was formally designated its official 'seat' at the Edinburgh meeting of the European Council of EU heads of state and government in December 1992. (This position was reconfirmed and given treaty status in the 1997 Treaty of Amsterdam). However, only the (four-day) plenary sessions of the Parliament are held in Strasbourg each month, with all other business being conducted in Brussels and Luxembourg. Those sessions take place in the Immeuble Louise Weiss, inaugurated in 1999, which houses the largest parliamentary assembly room in Europe and of any democratic institution in the world. Before that, the EP sessions had to take place in the main Council of Europe building, the Palace of Europe, whose unusual inner architecture had become a familiar sight to European TV audiences. In 1992, Strasbourg became the seat of the Franco-German TV channel and movie-production society Arte. +In addition to the cathedral, Strasbourg houses several other medieval churches that have survived the many wars and destructions that have plagued the city: the Romanesque Église Saint-Étienne, partly destroyed in 1944 by Allied bombing raids, the part Romanesque, part Gothic, very large Église Saint-Thomas with its Silbermann organ on which Wolfgang Amadeus Mozart and Albert Schweitzer played, the Gothic Église protestante Saint-Pierre-le-Jeune with its crypt dating back to the seventh century and its cloister partly from the eleventh century, the Gothic Église Saint-Guillaume with its fine early-Renaissance stained glass and furniture, the Gothic Église Saint-Jean, the part Gothic, part Art Nouveau Église Sainte-Madeleine, etc. The Neo-Gothic church Saint-Pierre-le-Vieux Catholique (there is also an adjacent church Saint-Pierre-le-Vieux Protestant) serves as a shrine for several 15th-century wood worked and painted altars coming from other, now destroyed churches and installed there for public display. Among the numerous secular medieval buildings, the monumental Ancienne Douane (old custom-house) stands out. +The German Renaissance has bequeathed the city some noteworthy buildings (especially the current Chambre de commerce et d'industrie, former town hall, on Place Gutenberg), as did the French Baroque and Classicism with several hôtels particuliers (i.e. palaces), among which the Palais Rohan (1742, now housing three museums) is the most spectacular. Other buildings of its kind are the "Hôtel de Hanau" (1736, now the city hall), the Hôtel de Klinglin (1736, now residence of the préfet), the Hôtel des Deux-Ponts (1755, now residence of the military governor), the Hôtel d'Andlau-Klinglin (1725, now seat of the administration of the Port autonome de Strasbourg) etc. The largest baroque building of Strasbourg though is the 150 m (490 ft) long 1720s main building of the Hôpital civil. As for French Neo-classicism, it is the Opera House on Place Broglie that most prestigiously represents this style. +Strasbourg also offers high-class eclecticist buildings in its very extended German district, the Neustadt, being the main memory of Wilhelmian architecture since most of the major cities in Germany proper suffered intensive damage during World War II. Streets, boulevards and avenues are homogeneous, surprisingly high (up to seven stories) and broad examples of German urban lay-out and of this architectural style that summons and mixes up five centuries of European architecture as well as Neo-Egyptian, Neo-Greek and Neo-Babylonian styles. The former imperial palace Palais du Rhin, the most political and thus heavily criticized of all German Strasbourg buildings epitomizes the grand scale and stylistic sturdiness of this period. But the two most handsome and ornate buildings of these times are the École internationale des Pontonniers (the former Höhere Mädchenschule, girls college) with its towers, turrets and multiple round and square angles and the École des Arts décoratifs with its lavishly ornate façade of painted bricks, woodwork and majolica. +As for modern and contemporary architecture, Strasbourg possesses some fine Art Nouveau buildings (such as the huge Palais des Fêtes and houses and villas like Villa Schutzenberger and Hôtel Brion), good examples of post-World War II functional architecture (the Cité Rotterdam, for which Le Corbusier did not succeed in the architectural contest) and, in the very extended Quartier Européen, some spectacular administrative buildings of sometimes utterly large size, among which the European Court of Human Rights building by Richard Rogers is arguably the finest. Other noticeable contemporary buildings are the new Music school Cité de la Musique et de la Danse, the Musée d'Art moderne et contemporain and the Hôtel du Département facing it, as well as, in the outskirts, the tramway-station Hoenheim-Nord designed by Zaha Hadid. +Strasbourg features a number of prominent parks, of which several are of cultural and historical interest: the Parc de l'Orangerie, laid out as a French garden by André le Nôtre and remodeled as an English garden on behalf of Joséphine de Beauharnais, now displaying noteworthy French gardens, a neo-classical castle and a small zoo; the Parc de la Citadelle, built around impressive remains of the 17th-century fortress erected close to the Rhine by Vauban; the Parc de Pourtalès, laid out in English style around a baroque castle (heavily restored in the 19th century) that now houses a small three-star hotel, and featuring an open-air museum of international contemporary sculpture. The Jardin botanique de l'Université de Strasbourg (botanical garden) was created under the German administration next to the Observatory of Strasbourg, built in 1881, and still owns some greenhouses of those times. The Parc des Contades, although the oldest park of the city, was completely remodeled after World War II. The futuristic Parc des Poteries is an example of European park-conception in the late 1990s. The Jardin des deux Rives, spread over Strasbourg and Kehl on both sides of the Rhine opened in 2004 and is the most extended (60-hectare) park of the agglomeration. The most recent park is Parc du Heyritz (8,7 ha), opened in 2014 along a canal facing the hôpital civil. +Unlike most other cities, Strasbourg's collections of European art are divided into several museums according not only to type and area, but also to epoch. Old master paintings from the Germanic Rhenish territories and until 1681 are displayed in the Musée de l'Œuvre Notre-Dame, old master paintings from all the rest of Europe (including the Dutch Rhenish territories) and until 1871 as well as old master paintings from the Germanic Rhenish territories between 1681 and 1871 are displayed in the Musée des Beaux-Arts. Old master graphic arts until 1871 is displayed in the Cabinet des estampes et dessins. Decorative arts until 1681 ("German period") are displayed in the Musée de l'Œuvre Notre-Dame, decorative arts from 1681 to 1871 ("French period") are displayed in the Musée des Arts décoratifs. International art (painting, sculpture, graphic arts) and decorative art since 1871 is displayed in the Musée d'art moderne et contemporain. The latter museum also displays the city's photographic library. +Strasbourg, well known as centre of humanism, has a long history of excellence in higher-education, at the crossroads of French and German intellectual traditions. Although Strasbourg had been annexed by the Kingdom of France in 1683, it still remained connected to the German-speaking intellectual world throughout the 18th century and the university attracted numerous students from the Holy Roman Empire, including Goethe, Metternich and Montgelas, who studied law in Strasbourg, among the most prominent. Nowadays, Strasbourg is known to offer among the best university courses in France, after Paris. +The Bibliothèque nationale et universitaire (BNU) is, with its collection of more than 3,000,000 titles, the second largest library in France after the Bibliothèque nationale de France. It was founded by the German administration after the complete destruction of the previous municipal library in 1871 and holds the unique status of being simultaneously a students' and a national library. The Strasbourg municipal library had been marked erroneously as "City Hall" in a French commercial map, which had been captured and used by the German artillery to lay their guns. A librarian from Munich later pointed out "...that the destruction of the precious collection was not the fault of a German artillery officer, who used the French map, but of the slovenly and inaccurate scholarship of a Frenchman." +As one of the earliest centers of book-printing in Europe (see above: History), Strasbourg for a long time held a large number of incunabula—documents printed before 1500—in her library as one of her most precious heritages. After the total destruction of this institution in 1870, however, a new collection had to be reassembled from scratch. Today, Strasbourg's different public and institutional libraries again display a sizable total number of incunabula, distributed as follows: Bibliothèque nationale et universitaire, ca. 2 098 Médiathèque de la ville et de la communauté urbaine de Strasbourg, 394 Bibliothèque du Grand Séminaire, 238 Médiathèque protestante, 94 and Bibliothèque alsatique du Crédit Mutuel, 5. +City transportation in Strasbourg includes the futurist-looking Strasbourg tramway that opened in 1994 and is operated by the regional transit company Compagnie des Transports Strasbourgeois (CTS), consisting of 6 lines with a total length of 55.8 km (34.7 mi). The CTS also operates a comprehensive bus network throughout the city that is integrated with the trams. With more than 500 km (311 mi) of bicycle paths, biking in the city is convenient and the CTS operates a cheap bike-sharing scheme named Vélhop'. The CTS, and its predecessors, also operated a previous generation of tram system between 1878 and 1960, complemented by trolleybus routes between 1939 and 1962. +Being a city on the Ill and close to the Rhine, Strasbourg has always been an important centre of fluvial navigation, as is attested by archeological findings. In 1682 the Canal de la Bruche was added to the river navigations, initially to provide transport for sandstone from quarries in the Vosges for use in the fortification of the city. That canal has since closed, but the subsequent Canal du Rhone au Rhine, Canal de la Marne au Rhin and Grand Canal d'Alsace are still in use, as is the important activity of the Port autonome de Strasbourg. Water tourism inside the city proper attracts hundreds of thousands of tourists yearly. +The tram system that now criss-crosses the historic city centre complements walking and biking in it. The centre has been transformed into a pedestrian priority zone that enables and invites walking and biking by making these active modes of transport comfortable, safe and enjoyable. These attributes are accomplished by applying the principle of "filtered permeability" to the existing irregular network of streets. It means that the network adaptations favour active transportation and, selectively, "filter out" the car by reducing the number of streets that run through the centre. While certain streets are discontinuous for cars, they connect to a network of pedestrian and bike paths which permeate the entire centre. In addition, these paths go through public squares and open spaces increasing the enjoyment of the trip. This logic of filtering a mode of transport is fully expressed in a comprehensive model for laying out neighbourhoods and districts – the Fused Grid. +At present the A35 autoroute, which parallels the Rhine between Karlsruhe and Basel, and the A4 autoroute, which links Paris with Strasbourg, penetrate close to the centre of the city. The Grand contournement ouest (GCO) project, programmed since 1999, plans to construct a 24 km (15 mi) long highway connection between the junctions of the A4 and the A35 autoroutes in the north and of the A35 and A352 autoroutes in the south. This routes well to the west of the city and is meant to divest a significant portion of motorized traffic from the unité urbaine. diff --git a/e2e/samples/test_documents/War_on_Terror.txt b/e2e/samples/test_documents/War_on_Terror.txt new file mode 100644 index 0000000000..1601e747ba --- /dev/null +++ b/e2e/samples/test_documents/War_on_Terror.txt @@ -0,0 +1,31 @@ +On 16 September 2001, at Camp David, President George W. Bush used the phrase war on terrorism in an unscripted and controversial comment when he said, "This crusade – this war on terrorism – is going to take a while, ... " Bush later apologized for this remark due to the negative connotations the term crusade has to people, e.g. of Muslim faith. The word crusade was not used again. On 20 September 2001, during a televised address to a joint session of congress, Bush stated that, "(o)ur 'war on terror' begins with al-Qaeda, but it does not end there. It will not end until every terrorist group of global reach has been found, stopped, and defeated." +U.S. President Barack Obama has rarely used the term, but in his inaugural address on 20 January 2009, he stated "Our nation is at war, against a far-reaching network of violence and hatred." In March 2009 the Defense Department officially changed the name of operations from "Global War on Terror" to "Overseas Contingency Operation" (OCO). In March 2009, the Obama administration requested that Pentagon staff members avoid use of the term, instead using "Overseas Contingency Operation". Basic objectives of the Bush administration "war on terror", such as targeting al Qaeda and building international counterterrorism alliances, remain in place. In December 2012, Jeh Johnson, the General Counsel of the Department of Defense, stated that the military fight will be replaced by a law enforcement operation when speaking at Oxford University, predicting that al Qaeda will be so weakened to be ineffective, and has been "effectively destroyed", and thus the conflict will not be an armed conflict under international law. In May 2013, Obama stated that the goal is "to dismantle specific networks of violent extremists that threaten America"; which coincided with the U.S. Office of Management and Budget having changed the wording from "Overseas Contingency Operations" to "Countering Violent Extremism" in 2010. +Because the actions involved in the "war on terrorism" are diffuse, and the criteria for inclusion are unclear, political theorist Richard Jackson has argued that "the 'war on terrorism' therefore, is simultaneously a set of actual practices—wars, covert operations, agencies, and institutions—and an accompanying series of assumptions, beliefs, justifications, and narratives—it is an entire language or discourse." Jackson cites among many examples a statement by John Ashcroft that "the attacks of September 11 drew a bright line of demarcation between the civil and the savage". Administration officials also described "terrorists" as hateful, treacherous, barbarous, mad, twisted, perverted, without faith, parasitical, inhuman, and, most commonly, evil. Americans, in contrast, were described as brave, loving, generous, strong, resourceful, heroic, and respectful of human rights. +The origins of al-Qaeda can be traced to the Soviet war in Afghanistan (December 1979 – February 1989). The United States, United Kingdom, Saudi Arabia, Pakistan, and the People's Republic of China supported the Islamist Afghan mujahadeen guerillas against the military forces of the Soviet Union and the Democratic Republic of Afghanistan. A small number of "Afghan Arab" volunteers joined the fight against the Soviets, including Osama bin Laden, but there is no evidence they received any external assistance. In May 1996 the group World Islamic Front for Jihad Against Jews and Crusaders (WIFJAJC), sponsored by bin Laden (and later re-formed as al-Qaeda), started forming a large base of operations in Afghanistan, where the Islamist extremist regime of the Taliban had seized power earlier in the year. In February 1998, Osama bin Laden signed a fatwā, as head of al-Qaeda, declaring war on the West and Israel, later in May of that same year al-Qaeda released a video declaring war on the U.S. and the West. +On 7 August 1998, al-Qaeda struck the U.S. embassies in Kenya and Tanzania, killing 224 people, including 12 Americans. In retaliation, U.S. President Bill Clinton launched Operation Infinite Reach, a bombing campaign in Sudan and Afghanistan against targets the U.S. asserted were associated with WIFJAJC, although others have questioned whether a pharmaceutical plant in Sudan was used as a chemical warfare plant. The plant produced much of the region's antimalarial drugs and around 50% of Sudan's pharmaceutical needs. The strikes failed to kill any leaders of WIFJAJC or the Taliban. +On the morning of 11 September 2001, 19 men affiliated with al-Qaeda hijacked four airliners all bound for California. Once the hijackers assumed control of the airliners, they told the passengers that they had the bomb on board and would spare the lives of passengers and crew once their demands were met – no passenger and crew actually suspected that they would use the airliners as suicide weapons since it had never happened before in history. The hijackers – members of al-Qaeda's Hamburg cell – intentionally crashed two airliners into the Twin Towers of the World Trade Center in New York City. Both buildings collapsed within two hours from fire damage related to the crashes, destroying nearby buildings and damaging others. The hijackers crashed a third airliner into the Pentagon in Arlington County, Virginia, just outside Washington D.C. The fourth plane crashed into a field near Shanksville, Pennsylvania, after some of its passengers and flight crew attempted to retake control of the plane, which the hijackers had redirected toward Washington D.C., to target the White House, or the U.S. Capitol. No flights had survivors. A total of 2,977 victims and the 19 hijackers perished in the attacks. +The Authorization for Use of Military Force Against Terrorists or "AUMF" was made law on 14 September 2001, to authorize the use of United States Armed Forces against those responsible for the attacks on 11 September 2001. It authorized the President to use all necessary and appropriate force against those nations, organizations, or persons he determines planned, authorized, committed, or aided the terrorist attacks that occurred on 11 September 2001, or harbored such organizations or persons, in order to prevent any future acts of international terrorism against the United States by such nations, organizations or persons. Congress declares this is intended to constitute specific statutory authorization within the meaning of section 5(b) of the War Powers Resolution of 1973. +Subsequently, in October 2001, U.S. forces (with UK and coalition allies) invaded Afghanistan to oust the Taliban regime. On 7 October 2001, the official invasion began with British and U.S. forces conducting airstrike campaigns over enemy targets. Kabul, the capital city of Afghanistan, fell by mid-November. The remaining al-Qaeda and Taliban remnants fell back to the rugged mountains of eastern Afghanistan, mainly Tora Bora. In December, Coalition forces (the U.S. and its allies) fought within that region. It is believed that Osama bin Laden escaped into Pakistan during the battle. +The Taliban regrouped in western Pakistan and began to unleash an insurgent-style offensive against Coalition forces in late 2002. Throughout southern and eastern Afghanistan, firefights broke out between the surging Taliban and Coalition forces. Coalition forces responded with a series of military offensives and an increase in the amount of troops in Afghanistan. In February 2010, Coalition forces launched Operation Moshtarak in southern Afghanistan along with other military offensives in the hopes that they would destroy the Taliban insurgency once and for all. Peace talks are also underway between Taliban affiliated fighters and Coalition forces. In September 2014, Afghanistan and the United States signed a security agreement, which permits United States and NATO forces to remain in Afghanistan until at least 2024. The United States and other NATO and non-NATO forces are planning to withdraw; with the Taliban claiming it has defeated the United States and NATO, and the Obama Administration viewing it as a victory. In December 2014, ISAF encasing its colors, and Resolute Support began as the NATO operation in Afghanistan. Continued United States operations within Afghanistan will continue under the name "Operation Freedom's Sentinel". +In January 2002, the United States Special Operations Command, Pacific deployed to the Philippines to advise and assist the Armed Forces of the Philippines in combating Filipino Islamist groups. The operations were mainly focused on removing the Abu Sayyaf group and Jemaah Islamiyah (JI) from their stronghold on the island of Basilan. The second portion of the operation was conducted as a humanitarian program called "Operation Smiles". The goal of the program was to provide medical care and services to the region of Basilan as part of a "Hearts and Minds" program. Joint Special Operations Task Force – Philippines disbanded in June 2014, ending a 14-year mission. After JSOTF-P disbanded, as late as November 2014, American forces continued to operate in the Philippines under the name "PACOM Augmentation Team". +On 14 September 2009, U.S. Special Forces killed two men and wounded and captured two others near the Somali village of Baarawe. Witnesses claim that helicopters used for the operation launched from French-flagged warships, but that could not be confirmed. A Somali-based al-Qaida affiliated group, the Al-Shabaab, has confirmed the death of "sheik commander" Saleh Ali Saleh Nabhan along with an unspecified number of militants. Nabhan, a Kenyan, was wanted in connection with the 2002 Mombasa attacks. +The conflict in northern Mali began in January 2012 with radical Islamists (affiliated to al-Qaeda) advancing into northern Mali. The Malian government had a hard time maintaining full control over their country. The fledgling government requested support from the international community on combating the Islamic militants. In January 2013, France intervened on behalf of the Malian government's request and deployed troops into the region. They launched Operation Serval on 11 January 2013, with the hopes of dislodging the al-Qaeda affiliated groups from northern Mali. +Following the ceasefire agreement that suspended hostilities (but not officially ended) in the 1991 Gulf War, the United States and its allies instituted and began patrolling Iraqi no-fly zones, to protect Iraq's Kurdish and Shi'a Arab population—both of which suffered attacks from the Hussein regime before and after the Gulf War—in Iraq's northern and southern regions, respectively. U.S. forces continued in combat zone deployments through November 1995 and launched Operation Desert Fox against Iraq in 1998 after it failed to meet U.S. demands of "unconditional cooperation" in weapons inspections. +The first ground attack came at the Battle of Umm Qasr on 21 March 2003 when a combined force of British, American and Polish forces seized control of the port city of Umm Qasr. Baghdad, Iraq's capital city, fell to American forces in April 2003 and Saddam Hussein's government quickly dissolved. On 1 May 2003, Bush announced that major combat operations in Iraq had ended. However, an insurgency arose against the U.S.-led coalition and the newly developing Iraqi military and post-Saddam government. The insurgency, which included al-Qaeda affiliated groups, led to far more coalition casualties than the invasion. Other elements of the insurgency were led by fugitive members of President Hussein's Ba'ath regime, which included Iraqi nationalists and pan-Arabists. Many insurgency leaders are Islamists and claim to be fighting a religious war to reestablish the Islamic Caliphate of centuries past. Iraq's former president, Saddam Hussein was captured by U.S. forces in December 2003. He was executed in 2006. +In a major split in the ranks of Al Qaeda's organization, the Iraqi franchise, known as Al Qaeda in Iraq covertly invaded Syria and the Levant and began participating in the ongoing Syrian Civil War, gaining enough support and strength to re-invade Iraq's western provinces under the name of the Islamic State of Iraq and the Levant (ISIS/ISIL), taking over much of the country in a blitzkrieg-like action and combining the Iraq insurgency and Syrian Civil War into a single conflict. Due to their extreme brutality and a complete change in their overall ideology, Al Qaeda's core organization in Central Asia eventually denounced ISIS and directed their affiliates to cut off all ties with this organization. Many analysts[who?] believe that because of this schism, Al Qaeda and ISIL are now in a competition to retain the title of the world's most powerful terrorist organization. +The Obama administration began to reengage in Iraq with a series of airstrikes aimed at ISIS beginning on 10 August 2014. On 9 September 2014 President Obama said that he had the authority he needed to take action to destroy the militant group known as the Islamic State of Iraq and the Levant, citing the 2001 Authorization for Use of Military Force Against Terrorists, and thus did not require additional approval from Congress. The following day on 10 September 2014 President Barack Obama made a televised speech about ISIL, which he stated "Our objective is clear: We will degrade, and ultimately destroy, ISIL through a comprehensive and sustained counter-terrorism strategy". Obama has authorized the deployment of additional U.S. Forces into Iraq, as well as authorizing direct military operations against ISIL within Syria. On the night of 21/22 September the United States, Saudi Arabia, Bahrain, the UAE, Jordan and Qatar started air attacks against ISIS in Syria.[citation needed] +Following the 11 September 2001 attacks, former President of Pakistan Pervez Musharraf sided with the U.S. against the Taliban government in Afghanistan after an ultimatum by then U.S. President George W. Bush. Musharraf agreed to give the U.S. the use of three airbases for Operation Enduring Freedom. United States Secretary of State Colin Powell and other U.S. administration officials met with Musharraf. On 19 September 2001, Musharraf addressed the people of Pakistan and stated that, while he opposed military tactics against the Taliban, Pakistan risked being endangered by an alliance of India and the U.S. if it did not cooperate. In 2006, Musharraf testified that this stance was pressured by threats from the U.S., and revealed in his memoirs that he had "war-gamed" the United States as an adversary and decided that it would end in a loss for Pakistan. +On 12 January 2002, Musharraf gave a speech against Islamic extremism. He unequivocally condemned all acts of terrorism and pledged to combat Islamic extremism and lawlessness within Pakistan itself. He stated that his government was committed to rooting out extremism and made it clear that the banned militant organizations would not be allowed to resurface under any new name. He said, "the recent decision to ban extremist groups promoting militancy was taken in the national interest after thorough consultations. It was not taken under any foreign influence". +In 2002, the Musharraf-led government took a firm stand against the jihadi organizations and groups promoting extremism, and arrested Maulana Masood Azhar, head of the Jaish-e-Mohammed, and Hafiz Muhammad Saeed, chief of the Lashkar-e-Taiba, and took dozens of activists into custody. An official ban was imposed on the groups on 12 January. Later that year, the Saudi born Zayn al-Abidn Muhammed Hasayn Abu Zubaydah was arrested by Pakistani officials during a series of joint U.S.-Pakistan raids. Zubaydah is said to have been a high-ranking al-Qaeda official with the title of operations chief and in charge of running al-Qaeda training camps. Other prominent al-Qaeda members were arrested in the following two years, namely Ramzi bin al-Shibh, who is known to have been a financial backer of al-Qaeda operations, and Khalid Sheikh Mohammed, who at the time of his capture was the third highest-ranking official in al-Qaeda and had been directly in charge of the planning for the 11 September attacks. +The use of drones by the Central Intelligence Agency in Pakistan to carry out operations associated with the Global War on Terror sparks debate over sovereignty and the laws of war. The U.S. Government uses the CIA rather than the U.S. Air Force for strikes in Pakistan in order to avoid breaching sovereignty through military invasion. The United States was criticized by[according to whom?] a report on drone warfare and aerial sovereignty for abusing the term 'Global War on Terror' to carry out military operations through government agencies without formally declaring war. +In a 'Letter to American People' written by Osama bin Laden in 2002, he stated that one of the reasons he was fighting America is because of its support of India on the Kashmir issue. While on a trip to Delhi in 2002, U.S. Secretary of Defense Donald Rumsfeld suggested that Al-Qaeda was active in Kashmir, though he did not have any hard evidence. An investigation in 2002 unearthed evidence that Al-Qaeda and its affiliates were prospering in Pakistan-administered Kashmir with tacit approval of Pakistan's National Intelligence agency Inter-Services Intelligence. A team of Special Air Service and Delta Force was sent into Indian-administered Kashmir in 2002 to hunt for Osama bin Laden after reports that he was being sheltered by the Kashmiri militant group Harkat-ul-Mujahideen. U.S. officials believed that Al-Qaeda was helping organize a campaign of terror in Kashmir in order to provoke conflict between India and Pakistan. Fazlur Rehman Khalil, the leader of the Harkat-ul-Mujahideen, signed al-Qaeda's 1998 declaration of holy war, which called on Muslims to attack all Americans and their allies. Indian sources claimed that In 2006, Al-Qaeda claimed they had established a wing in Kashmir; this worried the Indian government. India also claimed that Al-Qaeda has strong ties with the Kashmir militant groups Lashkar-e-Taiba and Jaish-e-Mohammed in Pakistan. While on a visit to Pakistan in January 2010, U.S. Defense secretary Robert Gates stated that Al-Qaeda was seeking to destabilize the region and planning to provoke a nuclear war between India and Pakistan. +In September 2009, a U.S. Drone strike reportedly killed Ilyas Kashmiri, who was the chief of Harkat-ul-Jihad al-Islami, a Kashmiri militant group associated with Al-Qaeda. Kashmiri was described by Bruce Riedel as a 'prominent' Al-Qaeda member, while others described him as the head of military operations for Al-Qaeda. Waziristan had now become the new battlefield for Kashmiri militants, who were now fighting NATO in support of Al-Qaeda. On 8 July 2012, Al-Badar Mujahideen, a breakaway faction of Kashmir centric terror group Hizbul Mujahideen, on conclusion of their two-day Shuhada Conference called for mobilisation of resources for continuation of jihad in Kashmir. +In the following months, NATO took a wide range of measures to respond to the threat of terrorism. On 22 November 2002, the member states of the Euro-Atlantic Partnership Council (EAPC) decided on a Partnership Action Plan against Terrorism, which explicitly states, "EAPC States are committed to the protection and promotion of fundamental freedoms and human rights, as well as the rule of law, in combating terrorism." NATO started naval operations in the Mediterranean Sea designed to prevent the movement of terrorists or weapons of mass destruction as well as to enhance the security of shipping in general called Operation Active Endeavour. +Support for the U.S. cooled when America made clear its determination to invade Iraq in late 2002. Even so, many of the "coalition of the willing" countries that unconditionally supported the U.S.-led military action have sent troops to Afghanistan, particular neighboring Pakistan, which has disowned its earlier support for the Taliban and contributed tens of thousands of soldiers to the conflict. Pakistan was also engaged in the War in North-West Pakistan (Waziristan War). Supported by U.S. intelligence, Pakistan was attempting to remove the Taliban insurgency and al-Qaeda element from the northern tribal areas. +The British 16th Air Assault Brigade (later reinforced by Royal Marines) formed the core of the force in southern Afghanistan, along with troops and helicopters from Australia, Canada and the Netherlands. The initial force consisted of roughly 3,300 British, 2,000 Canadian, 1,400 from the Netherlands and 240 from Australia, along with special forces from Denmark and Estonia and small contingents from other nations. The monthly supply of cargo containers through Pakistani route to ISAF in Afghanistan is over 4,000 costing around 12 billion in Pakistani Rupees. +In addition to military efforts abroad, in the aftermath of 9/11 the Bush Administration increased domestic efforts to prevent future attacks. Various government bureaucracies that handled security and military functions were reorganized. A new cabinet-level agency called the United States Department of Homeland Security was created in November 2002 to lead and coordinate the largest reorganization of the U.S. federal government since the consolidation of the armed forces into the Department of Defense.[citation needed] +The USA PATRIOT Act of October 2001 dramatically reduces restrictions on law enforcement agencies' ability to search telephone, e-mail communications, medical, financial, and other records; eases restrictions on foreign intelligence gathering within the United States; expands the Secretary of the Treasury's authority to regulate financial transactions, particularly those involving foreign individuals and entities; and broadens the discretion of law enforcement and immigration authorities in detaining and deporting immigrants suspected of terrorism-related acts. The act also expanded the definition of terrorism to include domestic terrorism, thus enlarging the number of activities to which the USA PATRIOT Act's expanded law enforcement powers could be applied. A new Terrorist Finance Tracking Program monitored the movements of terrorists' financial resources (discontinued after being revealed by The New York Times). Global telecommunication usage, including those with no links to terrorism, is being collected and monitored through the NSA electronic surveillance program. The Patriot Act is still in effect. +Political interest groups have stated that these laws remove important restrictions on governmental authority, and are a dangerous encroachment on civil liberties, possible unconstitutional violations of the Fourth Amendment. On 30 July 2003, the American Civil Liberties Union (ACLU) filed the first legal challenge against Section 215 of the Patriot Act, claiming that it allows the FBI to violate a citizen's First Amendment rights, Fourth Amendment rights, and right to due process, by granting the government the right to search a person's business, bookstore, and library records in a terrorist investigation, without disclosing to the individual that records were being searched. Also, governing bodies in a number of communities have passed symbolic resolutions against the act. +In 2005, the UN Security Council adopted Resolution 1624 concerning incitement to commit acts of terrorism and the obligations of countries to comply with international human rights laws. Although both resolutions require mandatory annual reports on counter-terrorism activities by adopting nations, the United States and Israel have both declined to submit reports. In the same year, the United States Department of Defense and the Chairman of the Joint Chiefs of Staff issued a planning document, by the name "National Military Strategic Plan for the War on Terrorism", which stated that it constituted the "comprehensive military plan to prosecute the Global War on Terror for the Armed Forces of the United States...including the findings and recommendations of the 9/11 Commission and a rigorous examination with the Department of Defense". +Criticism of the War on Terror addresses the issues, morality, efficiency, economics, and other questions surrounding the War on Terror and made against the phrase itself, calling it a misnomer. The notion of a "war" against "terrorism" has proven highly contentious, with critics charging that it has been exploited by participating governments to pursue long-standing policy/military objectives, reduce civil liberties, and infringe upon human rights. It is argued that the term war is not appropriate in this context (as in War on Drugs), since there is no identifiable enemy, and that it is unlikely international terrorism can be brought to an end by military means. +Other critics, such as Francis Fukuyama, note that "terrorism" is not an enemy, but a tactic; calling it a "war on terror", obscures differences between conflicts such as anti-occupation insurgents and international mujahideen. With a military presence in Iraq and Afghanistan and its associated collateral damage Shirley Williams maintains this increases resentment and terrorist threats against the West. There is also perceived U.S. hypocrisy, media-induced hysteria, and that differences in foreign and security policy have damaged America's image in most of the world. diff --git a/haystack/components/evaluators/document_mrr.py b/haystack/components/evaluators/document_mrr.py index f65cf8fe57..cf7e72fedc 100644 --- a/haystack/components/evaluators/document_mrr.py +++ b/haystack/components/evaluators/document_mrr.py @@ -53,7 +53,7 @@ def run( :returns: A dictionary with the following outputs: - `score` - The average of calculated scores. - - `invididual_scores` - A list of numbers from 0.0 to 1.0 that represents how high the first retrieved document is ranked. + - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents how high the first retrieved document is ranked. """ if len(ground_truth_documents) != len(retrieved_documents): msg = "The length of ground_truth_documents and retrieved_documents must be the same." diff --git a/pyproject.toml b/pyproject.toml index fca478aacb..adcd1adc59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -201,7 +201,7 @@ skip_magic_trailing_comma = true # For compatibility with pydoc>=4.6, check if [tool.codespell] ignore-words-list = "ans,astroid,nd,ned,nin,ue,rouge,ist" quiet-level = 3 -skip = "test/nodes/*,test/others/*,test/samples/*" +skip = "test/nodes/*,test/others/*,test/samples/*,e2e/*" [tool.pylint.'MESSAGES CONTROL'] max-line-length=120 From 554c6d7d28bddc525849398f41c2d7e3786023e7 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Thu, 25 Apr 2024 12:33:32 +0200 Subject: [PATCH 03/13] wip --- e2e/pipelines/test_evaluation_pipeline.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py index 36ee194b35..c639335dcc 100644 --- a/e2e/pipelines/test_evaluation_pipeline.py +++ b/e2e/pipelines/test_evaluation_pipeline.py @@ -8,7 +8,6 @@ DocumentMAPEvaluator, DocumentMRREvaluator, DocumentRecallEvaluator, - EvaluationResult, FaithfulnessEvaluator, SASEvaluator, ) @@ -188,7 +187,7 @@ def create_document(text: str, name: str): } ) - data = { + _ = { "inputs": { "question": all_questions, "contexts": all_contexts, @@ -229,6 +228,5 @@ def create_document(text: str, name: str): ], } - evaluation_result = EvaluationResult(pipeline_name="pipe_1", results=data) - - print(evaluation_result) + # evaluation_result = EvaluationResult(pipeline_name="pipe_1", results=data) + # print(evaluation_result) From a761cf9563dc8c345994a156b9e66daedf2c7c61 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Thu, 25 Apr 2024 16:03:10 +0200 Subject: [PATCH 04/13] cleaning up tests --- e2e/pipelines/test_evaluation_pipeline.py | 344 ++++++++++-------- e2e/samples/test_documents/Edmund_Burke.txt | 40 -- e2e/samples/test_documents/Freemasonry.txt | 37 -- e2e/samples/test_documents/John_Kerry.txt | 52 --- .../test_documents/Late_Middle_Ages.txt | 40 -- e2e/samples/test_documents/Normans.txt | 45 --- haystack/components/generators/chat/openai.py | 2 +- haystack/evaluation/eval_run_result.py | 2 +- proposals/text/4284-drop-basecomponent.md | 179 ++++----- ...sentence_transformers_embedding_backend.py | 2 +- 10 files changed, 283 insertions(+), 460 deletions(-) delete mode 100644 e2e/samples/test_documents/Edmund_Burke.txt delete mode 100644 e2e/samples/test_documents/Freemasonry.txt delete mode 100644 e2e/samples/test_documents/John_Kerry.txt delete mode 100644 e2e/samples/test_documents/Late_Middle_Ages.txt delete mode 100644 e2e/samples/test_documents/Normans.txt diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py index c639335dcc..5740297215 100644 --- a/e2e/pipelines/test_evaluation_pipeline.py +++ b/e2e/pipelines/test_evaluation_pipeline.py @@ -1,6 +1,8 @@ import os from typing import List +import pytest + from haystack import Document, Pipeline from haystack.components.builders import AnswerBuilder, PromptBuilder from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder @@ -17,15 +19,16 @@ from haystack.components.writers import DocumentWriter from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.document_stores.types import DuplicatePolicy +from haystack.evaluation import EvaluationRunResult +embeddings_model = "sentence-transformers/all-MiniLM-L6-v2" -def indexing(documents: List[Document]): - """ - Indexing the documents - """ + +def indexing_pipeline(documents: List[Document]): + """Indexing the documents""" document_store = InMemoryDocumentStore() doc_writer = DocumentWriter(document_store=document_store, policy=DuplicatePolicy.SKIP) - doc_embedder = SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2") + doc_embedder = SentenceTransformersDocumentEmbedder(model=embeddings_model) ingestion_pipe = Pipeline() ingestion_pipe.add_component(instance=doc_embedder, name="doc_embedder") ingestion_pipe.add_component(instance=doc_writer, name="doc_writer") @@ -34,10 +37,8 @@ def indexing(documents: List[Document]): return document_store -def build_rag_pipeline(document_store, top_k=2): - """ - Building the RAG pipeline - """ +def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int): + """Building the RAG pipeline""" template = """ You have to answer the following question based on the given context information only. @@ -49,184 +50,215 @@ def build_rag_pipeline(document_store, top_k=2): Question: {{question}} Answer: """ + rag_pipeline = Pipeline() + rag_pipeline.add_component("embedder", SentenceTransformersTextEmbedder(model=embeddings_model)) + rag_pipeline.add_component("retriever", InMemoryEmbeddingRetriever(document_store, top_k=top_k)) + rag_pipeline.add_component("prompt_builder", PromptBuilder(template=template)) + rag_pipeline.add_component("generator", OpenAIGenerator(model="gpt-3.5-turbo")) + rag_pipeline.add_component("answer_builder", AnswerBuilder()) + rag_pipeline.connect("embedder", "retriever.query_embedding") + rag_pipeline.connect("retriever", "prompt_builder.documents") + rag_pipeline.connect("prompt_builder", "generator") + rag_pipeline.connect("generator.replies", "answer_builder.replies") + rag_pipeline.connect("generator.meta", "answer_builder.meta") + rag_pipeline.connect("retriever", "answer_builder.documents") - rag_pipeline_1 = Pipeline() - rag_pipeline_1.add_component( - "query_embedder", SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2") - ) - rag_pipeline_1.add_component("retriever", InMemoryEmbeddingRetriever(document_store, top_k=top_k)) - rag_pipeline_1.add_component("prompt_builder", PromptBuilder(template=template)) - rag_pipeline_1.add_component("generator", OpenAIGenerator(model="gpt-3.5-turbo")) - rag_pipeline_1.add_component("answer_builder", AnswerBuilder()) + return rag_pipeline - rag_pipeline_1.connect("query_embedder", "retriever.query_embedding") - rag_pipeline_1.connect("retriever", "prompt_builder.documents") - rag_pipeline_1.connect("prompt_builder", "generator") - rag_pipeline_1.connect("generator.replies", "answer_builder.replies") - rag_pipeline_1.connect("generator.meta", "answer_builder.meta") - rag_pipeline_1.connect("retriever", "answer_builder.documents") - return rag_pipeline_1 +def evaluation_pipeline(questions, truth_docs, truth_answers, retrieved_docs, contexts, pred_answers): + """ + Run the evaluation pipeline + """ + eval_pipeline = Pipeline() + eval_pipeline.add_component("doc_mrr", DocumentMRREvaluator()) + eval_pipeline.add_component("groundness", FaithfulnessEvaluator()) + eval_pipeline.add_component("sas", SASEvaluator(model=embeddings_model)) + eval_pipeline.add_component("doc_map", DocumentMAPEvaluator()) + eval_pipeline.add_component("doc_recall_single_hit", DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT)) + eval_pipeline.add_component("doc_recall_multi_hit", DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT)) + return eval_pipeline.run( + { + "doc_mrr": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, + "groundness": {"questions": questions, "contexts": contexts, "responses": truth_answers}, + "sas": {"predicted_answers": pred_answers, "ground_truth_answers": truth_answers}, + "doc_map": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, + "doc_recall_single_hit": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, + "doc_recall_multi_hit": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, + } + ) -def test_evaluation_pipeline(samples_path): + +def run_rag_pipeline(documents, evaluation_questions, rag_pipeline_a): """ - Test the evaluation pipeline + Run the RAG pipeline and return the contexts, predicted answers, retrieved documents and ground truth documents """ - documents = [] - def create_document(text: str, name: str): - return Document(content=text, meta={"name": name}) + truth_docs = [] + retrieved_docs = [] + contexts = [] + pred_answers = [] - for root, dirs, files in os.walk(str(samples_path) + "/test_documents/"): - for article in files: - with open(f"{root}/{article}", "r") as f: - raw_texts = f.read().split("\n") - for text in raw_texts: - documents.append(create_document(text, article)) + for q in evaluation_questions: + response = rag_pipeline_a.run( + { + "embedder": {"text": q["question"]}, + "prompt_builder": {"question": q["question"]}, + "answer_builder": {"query": q["question"]}, + } + ) + truth_docs.append([doc for doc in documents if doc.meta["name"] in q["ground_truth_doc"]]) + retrieved_docs.append(response["answer_builder"]["answers"][0].documents) + contexts.append([doc.content for doc in response["answer_builder"]["answers"][0].documents]) + pred_answers.append(response["answer_builder"]["answers"][0].data) - document_store = indexing(documents) + return contexts, pred_answers, retrieved_docs, truth_docs - # collect all the data for evaluation - all_questions = [] - all_ground_truth_documents = [] - all_ground_truth_answers = [] - all_retrieved_documents = [] - all_contexts = [] - all_answers = [] - questions = [ - { - "question": "Who re-translated the Reflections into French?", - "answer": ["Louis XVI"], - "ground_truth_doc": ["Edmund_Burke.txt"], - }, - { - "question": "What was Kerry's role in the Yale Political Union as a junior?", - "answer": ["President of the Union"], - "ground_truth_doc": ["John_Kerry.txt"], - }, +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY", None), + reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.", +) +def test_evaluation_pipeline(samples_path): + """ + Test the evaluation pipeline + """ + docs = [] + articles = os.listdir(str(samples_path) + "/test_documents/") + for article in articles: + with open(f"{str(samples_path)}/test_documents/{article}", "r") as f: + for text in f.read().split("\n"): + docs.append(Document(content=text, meta={"name": article})) + doc_store = indexing_pipeline(docs) + + eval_questions = [ { "question": 'What falls within the term "cultural anthropology"?', - "answer": ["the ideology and analytical stance of cultural relativism"], + "answer": "the ideology and analytical stance of cultural relativism", "ground_truth_doc": ["Culture.txt"], }, { "question": "Who was the spiritual guide during the Protestant Reformation?", - "answer": ["Martin Bucer"], + "answer": "Martin Bucer", "ground_truth_doc": ["Strasbourg.txt"], }, - { - "question": "What separates many annelids' segments?", - "answer": ["Septa"], - "ground_truth_doc": ["Annelid.txt"], - }, + {"question": "What separates many annelids' segments?", "answer": "Septa", "ground_truth_doc": ["Annelid.txt"]}, { "question": "What is materialism?", - "answer": ["a form of philosophical monism"], + "answer": "a form of philosophical monism", "ground_truth_doc": ["Materialism.txt"], }, - { - "question": "Who did the Hungarian nobility elect as King of Hungary?", - "answer": ["Matthias"], - "ground_truth_doc": ["Late_Middle_Ages.txt"], - }, ] + questions = [q["question"] for q in eval_questions] + truth_answers = [q["answer"] for q in eval_questions] - rag_pipeline_1 = build_rag_pipeline(document_store, top_k=2) - - # ToDo: do this in batch to avoid multiple calls to the pipeline - for q in questions: - question = q["question"] - answer = q["answer"] - ground_truth_docs = [doc for doc in documents if doc.meta["name"] in q["ground_truth_doc"]] - all_ground_truth_documents.append(ground_truth_docs) - all_ground_truth_answers.append(answer[0]) - all_questions.append(question) - - response = rag_pipeline_1.run( - { - "query_embedder": {"text": question}, - "prompt_builder": {"question": question}, - "answer_builder": {"query": question}, - } - ) + rag_pipeline_a = rag_pipeline(doc_store, top_k=3) + contexts_a, pred_answers_a, retrieved_docs_a, truth_docs = run_rag_pipeline(docs, eval_questions, rag_pipeline_a) + results_rag_a = evaluation_pipeline( + questions, truth_docs, truth_answers, retrieved_docs_a, contexts_a, pred_answers_a + ) - all_retrieved_documents.append(response["answer_builder"]["answers"][0].documents) - all_contexts.append([doc.content for doc in response["answer_builder"]["answers"][0].documents]) - all_answers.append(response["answer_builder"]["answers"][0].data) + inputs_a = { + "question": questions, + "contexts": contexts_a, + "answer": truth_answers, + "predicted_answer": pred_answers_a, + } + results_a = { + "Mean Reciprocal Rank": { + "individual_scores": results_rag_a["doc_mrr"]["individual_scores"], + "score": results_rag_a["doc_mrr"]["score"], + }, + "Semantic Answer Similarity": { + "individual_scores": results_rag_a["sas"]["individual_scores"], + "score": results_rag_a["sas"]["score"], + }, + "Faithfulness": { + "individual_scores": results_rag_a["groundness"]["individual_scores"], + "score": results_rag_a["groundness"]["score"], + }, + "Document MAP": { + "individual_scores": results_rag_a["doc_map"]["individual_scores"], + "score": results_rag_a["doc_map"]["score"], + }, + "Document Recall Single Hit": { + "individual_scores": results_rag_a["doc_recall_single_hit"]["individual_scores"], + "score": results_rag_a["doc_recall_single_hit"]["score"], + }, + "Document Recall Multi Hit": { + "individual_scores": results_rag_a["doc_recall_multi_hit"]["individual_scores"], + "score": results_rag_a["doc_recall_multi_hit"]["score"], + }, + } + evaluation_result_a = EvaluationRunResult(run_name="rag_pipeline_a", results=results_a, inputs=inputs_a) + df_score_report = evaluation_result_a.score_report() - eval_pipeline = Pipeline() - eval_pipeline.add_component("doc_mrr", DocumentMRREvaluator()) - eval_pipeline.add_component("groundness", FaithfulnessEvaluator()) - eval_pipeline.add_component("sas", SASEvaluator(model="sentence-transformers/all-MiniLM-L6-v2")) - eval_pipeline.add_component("doc_map", DocumentMAPEvaluator()) - eval_pipeline.add_component("doc_recall_single_hit", DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT)) - eval_pipeline.add_component("doc_recall_multi_hit", DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT)) + assert len(df_score_report) == 6 + print(df_score_report.columns) + assert list(df_score_report.columns) == ["score"] + assert list(df_score_report.index) == [ + "Mean Reciprocal Rank", + "Semantic Answer Similarity", + "Faithfulness", + "Document MAP", + "Document Recall Single Hit", + "Document Recall Multi Hit", + ] + df = evaluation_result_a.to_pandas() + assert list(df.columns) == [ + "question", + "contexts", + "answer", + "predicted_answer", + "Mean Reciprocal Rank", + "Semantic Answer Similarity", + "Faithfulness", + "Document MAP", + "Document Recall Single Hit", + "Document Recall Multi Hit", + ] + assert len(df) == 4 - results = eval_pipeline.run( - { - "doc_mrr": { - "ground_truth_documents": all_ground_truth_documents, - "retrieved_documents": all_retrieved_documents, - }, - "groundness": {"questions": all_questions, "contexts": all_contexts, "responses": all_answers}, - "sas": {"predicted_answers": all_answers, "ground_truth_answers": all_ground_truth_answers}, - "doc_map": { - "ground_truth_documents": all_ground_truth_documents, - "retrieved_documents": all_retrieved_documents, - }, - "doc_recall_single_hit": { - "ground_truth_documents": all_ground_truth_documents, - "retrieved_documents": all_retrieved_documents, - }, - "doc_recall_multi_hit": { - "ground_truth_documents": all_ground_truth_documents, - "retrieved_documents": all_retrieved_documents, - }, - } + rag_pipeline_b = rag_pipeline(doc_store, top_k=5) + contexts_b, pred_answers_b, retrieved_docs_b, truth_docs = run_rag_pipeline(docs, eval_questions, rag_pipeline_b) + results_rag_b = evaluation_pipeline( + questions, truth_docs, truth_answers, retrieved_docs_b, contexts_b, pred_answers_b ) - _ = { - "inputs": { - "question": all_questions, - "contexts": all_contexts, - "answer": all_ground_truth_answers, - "predicted_answer": all_answers, + inputs_b = { + "question": questions, + "contexts": contexts_a, + "answer": truth_answers, + "predicted_answer": pred_answers_b, + } + results_b = { + "Mean Reciprocal Rank": { + "individual_scores": results_rag_b["doc_mrr"]["individual_scores"], + "score": results_rag_b["doc_mrr"]["score"], + }, + "Semantic Answer Similarity": { + "individual_scores": results_rag_b["sas"]["individual_scores"], + "score": results_rag_b["sas"]["score"], + }, + "Faithfulness": { + "individual_scores": results_rag_b["groundness"]["individual_scores"], + "score": results_rag_b["groundness"]["score"], + }, + "Document MAP": { + "individual_scores": results_rag_b["doc_map"]["individual_scores"], + "score": results_rag_b["doc_map"]["score"], + }, + "Document Recall Single Hit": { + "individual_scores": results_rag_b["doc_recall_single_hit"]["individual_scores"], + "score": results_rag_b["doc_recall_single_hit"]["score"], + }, + "Document Recall Multi Hit": { + "individual_scores": results_rag_b["doc_recall_multi_hit"]["individual_scores"], + "score": results_rag_b["doc_recall_multi_hit"]["score"], }, - "metrics": [ - { - "name": "Mean Reciprocal Rank", - "individual_scores": results["doc_mrr"]["individual_scores"], - "score": results["doc_mrr"]["score"], - }, - { - "name": "Semantic Answer Similarity", - "individual_scores": results["sas"]["individual_scores"], - "score": results["sas"]["score"], - }, - { - "name": "Faithfulness", - "individual_scores": results["groundness"]["individual_scores"], - "score": results["groundness"]["score"], - }, - { - "name": "Document MAP", - "individual_scores": results["doc_map"]["individual_scores"], - "score": results["doc_map"]["score"], - }, - { - "name": "Document Recall Single Hit", - "individual_scores": results["doc_recall_single_hit"]["individual_scores"], - "score": results["doc_recall_single_hit"]["score"], - }, - { - "name": "Document Recall Multi Hit", - "individual_scores": results["doc_recall_multi_hit"]["individual_scores"], - "score": results["doc_recall_multi_hit"]["score"], - }, - ], } + evaluation_result_b = EvaluationRunResult(run_name="rag_pipeline_b", results=results_b, inputs=inputs_b) + df_comparative = evaluation_result_a.comparative_individual_scores_report(evaluation_result_b) - # evaluation_result = EvaluationResult(pipeline_name="pipe_1", results=data) - # print(evaluation_result) + print(df_comparative) diff --git a/e2e/samples/test_documents/Edmund_Burke.txt b/e2e/samples/test_documents/Edmund_Burke.txt deleted file mode 100644 index 47ff51559d..0000000000 --- a/e2e/samples/test_documents/Edmund_Burke.txt +++ /dev/null @@ -1,40 +0,0 @@ -Burke was born in Dublin, Ireland. His mother Mary née Nagle (c. 1702 – 1770) was a Roman Catholic who hailed from a déclassé County Cork family (and a cousin of Nano Nagle), whereas his father, a successful solicitor, Richard (died 1761), was a member of the Church of Ireland; it remains unclear whether this is the same Richard Burke who converted from Catholicism. The Burke dynasty descends from an Anglo-Norman knight surnamed de Burgh (latinised as de Burgo) who arrived in Ireland in 1185 following Henry II of England's 1171 invasion of Ireland. -In 1744, Burke started at Trinity College Dublin, a Protestant establishment, which up until 1793, did not permit Catholics to take degrees. In 1747, he set up a debating society, "Edmund Burke's Club", which, in 1770, merged with TCD's Historical Club to form the College Historical Society; it is the oldest undergraduate society in the world. The minutes of the meetings of Burke's Club remain in the collection of the Historical Society. Burke graduated from Trinity in 1748. Burke's father wanted him to read Law, and with this in mind he went to London in 1750, where he entered the Middle Temple, before soon giving up legal study to travel in Continental Europe. After eschewing the Law, he pursued a livelihood through writing. -Burke claimed that Bolingbroke's arguments against revealed religion could apply to all social and civil institutions as well. Lord Chesterfield and Bishop Warburton (and others) initially thought that the work was genuinely by Bolingbroke rather than a satire. All the reviews of the work were positive, with critics especially appreciative of Burke's quality of writing. Some reviewers failed to notice the ironic nature of the book, which led to Burke stating in the preface to the second edition (1757) that it was a satire. -Richard Hurd believed that Burke's imitation was near-perfect and that this defeated his purpose: an ironist "should take care by a constant exaggeration to make the ridicule shine through the Imitation. Whereas this Vindication is everywhere enforc'd, not only in the language, and on the principles of L. Bol., but with so apparent, or rather so real an earnestness, that half his purpose is sacrificed to the other". A minority of scholars have taken the position that, in fact, Burke did write the Vindication in earnest, later disowning it only for political reasons. -On 25 February 1757, Burke signed a contract with Robert Dodsley to write a "history of England from the time of Julius Caesar to the end of the reign of Queen Anne", its length being eighty quarto sheets (640 pages), nearly 400,000 words. It was to be submitted for publication by Christmas 1758. Burke completed the work to the year 1216 and stopped; it was not published until after Burke's death, being included in an 1812 collection of his works, entitled An Essay Towards an Abridgement of the English History. G. M. Young did not value Burke's history and claimed that it was "demonstrably a translation from the French". Lord Acton, on commenting on the story that Burke stopped his history because David Hume published his, said "it is ever to be regretted that the reverse did not occur". -During the year following that contract, with Dodsley, Burke founded the influential Annual Register, a publication in which various authors evaluated the international political events of the previous year. The extent to which Burke contributed to the Annual Register is unclear: in his biography of Burke, Robert Murray quotes the Register as evidence of Burke's opinions, yet Philip Magnus in his biography does not cite it directly as a reference. Burke remained the chief editor of the publication until at least 1789 and there is no evidence that any other writer contributed to it before 1766. -At about this same time, Burke was introduced to William Gerard Hamilton (known as "Single-speech Hamilton"). When Hamilton was appointed Chief Secretary for Ireland, Burke accompanied him to Dublin as his private secretary, a position he held for three years. In 1765 Burke became private secretary to the liberal Whig statesman, Charles, Marquess of Rockingham, then Prime Minister of Great Britain, who remained Burke's close friend and associate until his untimely death in 1782. Rockingham also introduced Burke as a Freemason. -Burke took a leading role in the debate regarding the constitutional limits to the executive authority of the king. He argued strongly against unrestrained royal power and for the role of political parties in maintaining a principled opposition capable of preventing abuses, either by the monarch, or by specific factions within the government. His most important publication in this regard was his Thoughts on the Cause of the Present Discontents of 23 April 1770. Burke identified the "discontents" as stemming from the "secret influence" of a neo-Tory group he labelled as, the "king's friends", whose system "comprehending the exterior and interior administrations, is commonly called, in the technical language of the Court, Double Cabinet". Britain needed a party with "an unshaken adherence to principle, and attachment to connexion, against every allurement of interest". Party divisions "whether operating for good or evil, are things inseparable from free government". -In May 1778, Burke supported a parliamentary motion revising restrictions on Irish trade. His constituents, citizens of the great trading city of Bristol, however urged Burke to oppose free trade with Ireland. Burke resisted their protestations and said: "If, from this conduct, I shall forfeit their suffrages at an ensuing election, it will stand on record an example to future representatives of the Commons of England, that one man at least had dared to resist the desires of his constituents when his judgment assured him they were wrong". -Burke was not merely presenting a peace agreement to Parliament; rather, he stepped forward with four reasons against using force, carefully reasoned. He laid out his objections in an orderly manner, focusing on one before moving to the next. His first concern was that the use of force would have to be temporary, and that the uprisings and objections to British governance in America would not be. Second, Burke worried about the uncertainty surrounding whether Britain would win a conflict in America. "An armament", Burke said, "is not a victory". Third, Burke brought up the issue of impairment; it would do the British Government no good to engage in a scorched earth war and have the object they desired (America) become damaged or even useless. The American colonists could always retreat into the mountains, but the land they left behind would most likely be unusable, whether by accident or design. The fourth and final reason to avoid the use of force was experience; the British had never attempted to rein in an unruly colony by force, and they did not know if it could be done, let alone accomplished thousands of miles away from home. Not only were all of these concerns reasonable, but some turned out to be prophetic – the American colonists did not surrender, even when things looked extremely bleak, and the British were ultimately unsuccessful in their attempts to win a war fought on American soil. -Among the reasons this speech was so greatly admired was its passage on Lord Bathurst (1684–1775); Burke describes an angel in 1704 prophesying to Bathurst the future greatness of England and also of America: "Young man, There is America – which at this day serves little more than to amuse you with stories of savage men, and uncouth manners; yet shall, before you taste of death, shew itself equal to the whole of that commerce which now attracts the envy of the world". Samuel Johnson was so irritated at hearing it continually praised, that he made a parody of it, where the devil appears to a young Whig and predicts that in short time, Whiggism will poison even the paradise of America! -The administration of Lord North (1770–1782) tried to defeat the colonist rebellion by military force. British and American forces clashed in 1775 and, in 1776, came the American Declaration of Independence. Burke was appalled by celebrations in Britain of the defeat of the Americans at New York and Pennsylvania. He claimed the English national character was being changed by this authoritarianism. Burke wrote: "As to the good people of England, they seem to partake every day more and more of the Character of that administration which they have been induced to tolerate. I am satisfied, that within a few years there has been a great Change in the National Character. We seem no longer that eager, inquisitive, jealous, fiery people, which we have been formerly". -The Paymaster General Act 1782 ended the post as a lucrative sinecure. Previously, Paymasters had been able to draw on money from HM Treasury at their discretion. Now they were required to put the money they had requested to withdraw from the Treasury into the Bank of England, from where it was to be withdrawn for specific purposes. The Treasury would receive monthly statements of the Paymaster's balance at the Bank. This act was repealed by Shelburne's administration, but the act that replaced it repeated verbatim almost the whole text of the Burke Act. -Burke was a leading sceptic with respect to democracy. While admitting that theoretically, in some cases it might be desirable, he insisted a democratic government in Britain in his day would not only be inept, but also oppressive. He opposed democracy for three basic reasons. First, government required a degree of intelligence and breadth of knowledge of the sort that occurred rarely among the common people. Second, he thought that if they had the vote, common people had dangerous and angry passions that could be aroused easily by demagogues; he feared that the authoritarian impulses that could be empowered by these passions would undermine cherished traditions and established religion, leading to violence and confiscation of property. Third, Burke warned that democracy would create a tyranny over unpopular minorities, who needed the protection of the upper classes. -For years Burke pursued impeachment efforts against Warren Hastings, formerly Governor-General of Bengal, that resulted in the trial during 1786. His interaction with the British dominion of India began well before Hastings' impeachment trial. For two decades prior to the impeachment, Parliament had dealt with the Indian issue. This trial was the pinnacle of years of unrest and deliberation. In 1781 Burke was first able to delve into the issues surrounding the East India Company when he was appointed Chairman of the Commons Select Committee on East Indian Affairs—from that point until the end of the trial; India was Burke's primary concern. This committee was charged "to investigate alleged injustices in Bengal, the war with Hyder Ali, and other Indian difficulties". While Burke and the committee focused their attention on these matters, a second 'secret' committee was formed to assess the same issues. Both committee reports were written by Burke. Among other purposes, the reports conveyed to the Indian princes that Britain would not wage war on them, along with demanding that the HEIC recall Hastings. This was Burke's first call for substantive change regarding imperial practices. When addressing the whole House of Commons regarding the committee report, Burke described the Indian issue as one that "began 'in commerce' but 'ended in empire.'" -On 4 April 1786, Burke presented the Commons with the Article of Charge of High Crimes and Misdemeanors against Hastings. The impeachment in Westminster Hall, which did not begin until 14 February 1788, would be the "first major public discursive event of its kind in England", bringing the morality and duty of imperialism to the forefront of public perception. Burke already was known for his eloquent rhetorical skills and his involvement in the trial only enhanced its popularity and significance. Burke's indictment, fuelled by emotional indignation, branded Hastings a 'captain-general of iniquity'; who never dined without 'creating a famine'; whose heart was 'gangrened to the core', and who resembled both a 'spider of Hell' and a 'ravenous vulture devouring the carcasses of the dead'. The House of Commons eventually impeached Hastings, but subsequently, the House of Lords acquitted him of all charges. -Initially, Burke did not condemn the French Revolution. In a letter of 9 August 1789, Burke wrote: "England gazing with astonishment at a French struggle for Liberty and not knowing whether to blame or to applaud! The thing indeed, though I thought I saw something like it in progress for several years, has still something in it paradoxical and Mysterious. The spirit it is impossible not to admire; but the old Parisian ferocity has broken out in a shocking manner". The events of 5–6 October 1789, when a crowd of Parisian women marched on Versailles to compel King Louis XVI to return to Paris, turned Burke against it. In a letter to his son, Richard Burke, dated 10 October he said: "This day I heard from Laurence who has sent me papers confirming the portentous state of France—where the Elements which compose Human Society seem all to be dissolved, and a world of Monsters to be produced in the place of it—where Mirabeau presides as the Grand Anarch; and the late Grand Monarch makes a figure as ridiculous as pitiable". On 4 November Charles-Jean-François Depont wrote to Burke, requesting that he endorse the Revolution. Burke replied that any critical language of it by him should be taken "as no more than the expression of doubt" but he added: "You may have subverted Monarchy, but not recover'd freedom". In the same month he described France as "a country undone". Burke's first public condemnation of the Revolution occurred on the debate in Parliament on the army estimates on 9 February 1790, provoked by praise of the Revolution by Pitt and Fox: -In January 1790, Burke read Dr. Richard Price's sermon of 4 November 1789 entitled, A Discourse on the Love of our Country, to the Revolution Society. That society had been founded to commemorate the Glorious Revolution of 1688. In this sermon Price espoused the philosophy of universal "Rights of Men". Price argued that love of our country "does not imply any conviction of the superior value of it to other countries, or any particular preference of its laws and constitution of government". Instead, Price asserted that Englishmen should see themselves "more as citizens of the world than as members of any particular community". -Immediately after reading Price's sermon, Burke wrote a draft of what eventually became, Reflections on the Revolution in France. On 13 February 1790, a notice in the press said that shortly, Burke would publish a pamphlet on the Revolution and its British supporters, however he spent the year revising and expanding it. On 1 November he finally published the Reflections and it was an immediate best-seller. Priced at five shillings, it was more expensive than most political pamphlets, but by the end of 1790, it had gone through ten printings and sold approximately 17,500 copies. A French translation appeared on 29 November and on 30 November the translator, Pierre-Gaëton Dupont, wrote to Burke saying 2,500 copies had already been sold. The French translation ran to ten printings by June 1791. -Burke put forward that "We fear God, we look up with awe to kings; with affection to parliaments; with duty to magistrates; with reverence to priests; and with respect to nobility. Why? Because when such ideas are brought before our minds, it is natural to be so affected". Burke defended this prejudice on the grounds that it is "the general bank and capital of nations, and of ages" and superior to individual reason, which is small in comparison. "Prejudice", Burke claimed, "is of ready application in the emergency; it previously engages the mind in a steady course of wisdom and virtue, and does not leave the man hesitating in the moment of decision, skeptical, puzzled, and unresolved. Prejudice renders a man's virtue his habit". Burke criticised social contract theory by claiming that society is indeed, a contract, but "a partnership not only between those who are living, but between those who are living, those who are dead, and those who are to be born". -The most famous passage in Burke's Reflections was his description of the events of 5–6 October 1789 and the part of Marie-Antoinette in them. Burke's account differs little from modern historians who have used primary sources. His use of flowery language to describe it, however, provoked both praise and criticism. Philip Francis wrote to Burke saying that what he wrote of Marie-Antoinette was "pure foppery". Edward Gibbon, however, reacted differently: "I adore his chivalry". Burke was informed by an Englishman who had talked with the Duchesse de Biron, that when Marie-Antoinette was reading the passage, she burst into tears and took considerable time to finish reading it. Price had rejoiced that the French king had been "led in triumph" during the October Days, but to Burke this symbolised the opposing revolutionary sentiment of the Jacobins and the natural sentiments of those who shared his own view with horror—that the ungallant assault on Marie-Antoinette—was a cowardly attack on a defenceless woman. -Louis XVI translated the Reflections "from end to end" into French. Fellow Whig MPs Richard Sheridan and Charles James Fox, disagreed with Burke and split with him. Fox thought the Reflections to be "in very bad taste" and "favouring Tory principles". Other Whigs such as the Duke of Portland and Earl Fitzwilliam privately agreed with Burke, but did not wish for a public breach with their Whig colleagues. Burke wrote on 29 November 1790: "I have received from the Duke of Portland, Lord Fitzwilliam, the Duke of Devonshire, Lord John Cavendish, Montagu (Frederick Montagu MP), and a long et cetera of the old Stamina of the Whiggs a most full approbation of the principles of that work and a kind indulgence to the execution". The Duke of Portland said in 1791 that when anyone criticised the Reflections to him, he informed them that he had recommended the book to his sons as containing the true Whig creed. -Burke's Reflections sparked a pamphlet war. Thomas Paine penned the Rights of Man in 1791 as a response to Burke; Mary Wollstonecraft published A Vindication of the Rights of Men and James Mackintosh wrote Vindiciae Gallicae. Mackintosh was the first to see the Reflections as "the manifesto of a Counter Revolution". Mackintosh later agreed with Burke's views, remarking in December 1796 after meeting him, that Burke was "minutely and accurately informed, to a wonderful exactness, with respect to every fact relating to the French Revolution". Mackintosh later said: "Burke was one of the first thinkers as well as one of the greatest orators of his time. He is without parallel in any age, excepting perhaps Lord Bacon and Cicero; and his works contain an ampler store of political and moral wisdom than can be found in any other writer whatever". -In November 1790, François-Louis-Thibault de Menonville, a member of the National Assembly of France, wrote to Burke, praising Reflections and requesting more "very refreshing mental food" that he could publish. This Burke did in April 1791 when he published A Letter to a Member of the National Assembly. Burke called for external forces to reverse the revolution and included an attack on the late French philosopher Jean-Jacques Rousseau, as being the subject of a personality cult that had developed in revolutionary France. Although Burke conceded that Rousseau sometimes showed "a considerable insight into human nature" he mostly was critical. Although he did not meet Rousseau on his visit to Britain in 1766–7 Burke was a friend of David Hume, with whom Rousseau had stayed. Burke said Rousseau "entertained no principle either to influence of his heart, or to guide his understanding—but vanity"—which he "was possessed to a degree little short of madness". He also cited Rousseau's Confessions as evidence that Rousseau had a life of "obscure and vulgar vices" that was not "chequered, or spotted here and there, with virtues, or even distinguished by a single good action". Burke contrasted Rousseau's theory of universal benevolence and his having sent his children to a foundling hospital: "a lover of his kind, but a hater of his kindred". -These events and the disagreements that arose from them within the Whig Party, led to its break-up and to the rupture of Burke's friendship with Fox. In debate in Parliament on Britain's relations with Russia, Fox praised the principles of the revolution, although Burke was not able to reply at this time as he was "overpowered by continued cries of question from his own side of the House". When Parliament was debating the Quebec Bill for a constitution for Canada, Fox praised the revolution and criticised some of Burke's arguments, such as hereditary power. On 6 May 1791, during another debate in Parliament on the Quebec Bill, Burke used the opportunity to answer Fox, and to condemn the new French Constitution and "the horrible consequences flowing from the French idea of the Rights of Man". Burke asserted that those ideas were the antithesis of both the British and the American constitutions. Burke was interrupted, and Fox intervened, saying that Burke should be allowed to carry on with his speech. A vote of censure was moved against Burke, however, for noticing the affairs of France, which was moved by Lord Sheffield and seconded by Fox. Pitt made a speech praising Burke, and Fox made a speech—both rebuking and complimenting Burke. He questioned the sincerity of Burke, who seemed to have forgotten the lessons he had learned from him, quoting from Burke's own speeches of fourteen and fifteen years before. -At this point, Fox whispered that there was "no loss of friendship". "I regret to say there is", Burke replied, "I have indeed made a great sacrifice; I have done my duty though I have lost my friend. There is something in the detested French constitution that envenoms every thing it touches". This provoked a reply from Fox, yet he was unable to give his speech for some time since he was overcome with tears and emotion, he appealed to Burke to remember their inalienable friendship, but also repeated his criticisms of Burke and uttered "unusually bitter sarcasms". This only aggravated the rupture between the two men. Burke demonstrated his separation from the party on 5 June 1791 by writing to Fitzwilliam, declining money from him. -Burke knew that many members of the Whig Party did not share Fox's views and he wanted to provoke them into condemning the French Revolution. Burke wrote that he wanted to represent the whole Whig party "as tolerating, and by a toleration, countenancing those proceedings" so that he could "stimulate them to a public declaration of what every one of their acquaintance privately knows to be...their sentiments". Therefore, on 3 August 1791 Burke published his Appeal from the New to the Old Whigs, in which he renewed his criticism of the radical revolutionary programmes inspired by the French Revolution and attacked the Whigs who supported them, as holding principles contrary to those traditionally held by the Whig party. -Although Whig grandees such as Portland and Fitzwilliam privately agreed with Burke's Appeal, they wished he had used more moderate language. Fitzwilliam saw the Appeal as containing "the doctrines I have sworn by, long and long since". Francis Basset, a backbench Whig MP, wrote to Burke: "...though for reasons which I will not now detail I did not then deliver my sentiments, I most perfectly differ from Mr. Fox & from the great Body of opposition on the French Revolution". Burke sent a copy of the Appeal to the king and the king requested a friend to communicate to Burke that he had read it "with great Satisfaction". Burke wrote of its reception: "Not one word from one of our party. They are secretly galled. They agree with me to a title; but they dare not speak out for fear of hurting Fox. ... They leave me to myself; they see that I can do myself justice". Charles Burney viewed it as "a most admirable book—the best & most useful on political subjects that I have ever seen" but believed the differences in the Whig Party between Burke and Fox should not be aired publicly. -Burke supported the war against revolutionary France, seeing Britain as fighting on the side of the royalists and émigres in a civil war, rather than fighting against the whole nation of France. Burke also supported the royalist uprising in La Vendée, describing it on 4 November 1793 in a letter to William Windham, as "the sole affair I have much heart in". Burke wrote to Henry Dundas on 7 October urging him to send reinforcements there, as he viewed it as the only theatre in the war that might lead to a march on Paris. Dundas did not follow Burke's advice, however. -Burke believed the Government was not taking the uprising seriously enough, a view reinforced by a letter he had received from the Prince Charles of France (S.A.R. le comte d'Artois), dated 23 October, requesting that he intercede on behalf of the royalists to the Government. Burke was forced to reply on 6 November: "I am not in His Majesty's Service; or at all consulted in his Affairs". Burke published his Remarks on the Policy of the Allies with Respect to France, begun in October, where he said: "I am sure every thing has shewn us that in this war with France, one Frenchman is worth twenty foreigners. La Vendée is a proof of this". -On 20 June 1794, Burke received a vote of thanks from the Commons for his services in the Hastings Trial and he immediately resigned his seat, being replaced by his son Richard. A tragic blow fell upon Burke with the loss of Richard in August 1794, to whom he was tenderly attached, and in whom he saw signs of promise, which were not patent to others and which, in fact, appear to have been non-existent (though this view may have rather reflected the fact that Richard Burke had worked successfully in the early battle for Catholic emancipation). King George III, whose favour he had gained by his attitude on the French Revolution, wished to create him Earl of Beaconsfield, but the death of his son deprived the opportunity of such an honour and all its attractions, so the only award he would accept was a pension of £2,500. Even this modest reward was attacked by the Duke of Bedford and the Earl of Lauderdale, to whom Burke replied in his Letter to a Noble Lord (1796): "It cannot at this time be too often repeated; line upon line; precept upon precept; until it comes into the currency of a proverb, To innovate is not to reform". He argued that he was rewarded on merit, but the Duke of Bedford received his rewards from inheritance alone, his ancestor being the original pensioner: "Mine was from a mild and benevolent sovereign; his from Henry the Eighth". Burke also hinted at what would happen to such people if their revolutionary ideas were implemented, and included a description of the British constitution: -Burke's last publications were the Letters on a Regicide Peace (October 1796), called forth by negotiations for peace with France by the Pitt government. Burke regarded this as appeasement, injurious to national dignity and honour. In his Second Letter, Burke wrote of the French Revolutionary Government: "Individuality is left out of their scheme of government. The State is all in all. Everything is referred to the production of force; afterwards, everything is trusted to the use of it. It is military in its principle, in its maxims, in its spirit, and in all its movements. The State has dominion and conquest for its sole objects—dominion over minds by proselytism, over bodies by arms". -This is held to be the first explanation of the modern concept of totalitarian state. Burke regarded the war with France as ideological, against an "armed doctrine". He wished that France would not be partitioned due to the effect this would have on the balance of power in Europe, and that the war was not against France, but against the revolutionaries governing her. Burke said: "It is not France extending a foreign empire over other nations: it is a sect aiming at universal empire, and beginning with the conquest of France". -In November 1795, there was a debate in Parliament on the high price of corn and Burke wrote a memorandum to Pitt on the subject. In December Samuel Whitbread MP introduced a bill giving magistrates the power to fix minimum wages and Fox said he would vote for it. This debate probably led Burke to editing his memorandum, as there appeared a notice that Burke would soon publish a letter on the subject to the Secretary of the Board of Agriculture, Arthur Young; but he failed to complete it. These fragments were inserted into the memorandum after his death and published posthumously in 1800 as, Thoughts and Details on Scarcity. In it, Burke expounded "some of the doctrines of political economists bearing upon agriculture as a trade". Burke criticised policies such as maximum prices and state regulation of wages, and set out what the limits of government should be: -Writing to a friend in May 1795, Burke surveyed the causes of discontent: "I think I can hardly overrate the malignity of the principles of Protestant ascendency, as they affect Ireland; or of Indianism [i.e. corporate tyranny, as practiced by the British East Indies Company], as they affect these countries, and as they affect Asia; or of Jacobinism, as they affect all Europe, and the state of human society itself. The last is the greatest evil". By March 1796, however Burke had changed his mind: "Our Government and our Laws are beset by two different Enemies, which are sapping its foundations, Indianism, and Jacobinism. In some Cases they act separately, in some they act in conjunction: But of this I am sure; that the first is the worst by far, and the hardest to deal with; and for this amongst other reasons, that it weakens discredits, and ruins that force, which ought to be employed with the greatest Credit and Energy against the other; and that it furnishes Jacobinism with its strongest arms against all formal Government". -Burke believed that property was essential to human life. Because of his conviction that people desire to be ruled and controlled, the division of property formed the basis for social structure, helping develop control within a property-based hierarchy. He viewed the social changes brought on by property as the natural order of events, which should be taking place as the human race progressed. With the division of property and the class system, he also believed that it kept the monarch in check to the needs of the classes beneath the monarch. Since property largely aligned or defined divisions of social class, class too, was seen as natural—part of a social agreement that the setting of persons into different classes, is the mutual benefit of all subjects. Concern for property is not Burke's only influence. As Christopher Hitchens summarises, "If modern conservatism can be held to derive from Burke, it is not just because he appealed to property owners in behalf of stability but also because he appealed to an everyday interest in the preservation of the ancestral and the immemorial." -In the nineteenth century Burke was praised by both liberals and conservatives. Burke's friend Philip Francis wrote that Burke "was a man who truly & prophetically foresaw all the consequences which would rise from the adoption of the French principles" but because Burke wrote with so much passion, people were doubtful of his arguments. William Windham spoke from the same bench in the House of Commons as Burke had, when he had separated from Fox, and an observer said Windham spoke "like the ghost of Burke" when he made a speech against peace with France in 1801. William Hazlitt, a political opponent of Burke, regarded him as amongst his three favourite writers (the others being Junius and Rousseau), and made it "a test of the sense and candour of any one belonging to the opposite party, whether he allowed Burke to be a great man". William Wordsworth was originally a supporter of the French Revolution and attacked Burke in 'A Letter to the Bishop of Llandaff' (1793), but by the early nineteenth century he had changed his mind and came to admire Burke. In his Two Addresses to the Freeholders of Westmorland Wordsworth called Burke "the most sagacious Politician of his age" whose predictions "time has verified". He later revised his poem The Prelude to include praise of Burke ("Genius of Burke! forgive the pen seduced/By specious wonders") and portrayed him as an old oak. Samuel Taylor Coleridge came to have a similar conversion: he had criticised Burke in The Watchman, but in his Friend (1809–10) Coleridge defended Burke from charges of inconsistency. Later, in his Biographia Literaria (1817) Coleridge hails Burke as a prophet and praises Burke for referring "habitually to principles. He was a scientific statesman; and therefore a seer". Henry Brougham wrote of Burke: "... all his predictions, save one momentary expression, had been more than fulfilled: anarchy and bloodshed had borne sway in France; conquest and convulsion had desolated Europe...the providence of mortals is not often able to penetrate so far as this into futurity". George Canning believed that Burke's Reflections "has been justified by the course of subsequent events; and almost every prophecy has been strictly fulfilled". In 1823 Canning wrote that he took Burke's "last works and words [as] the manual of my politics". The Conservative Prime Minister Benjamin Disraeli "was deeply penetrated with the spirit and sentiment of Burke's later writings". -The 19th-century Liberal Prime Minister William Ewart Gladstone considered Burke "a magazine of wisdom on Ireland and America" and in his diary recorded: "Made many extracts from Burke—sometimes almost divine". The Radical MP and anti-Corn Law activist Richard Cobden often praised Burke's Thoughts and Details on Scarcity. The Liberal historian Lord Acton considered Burke one of the three greatest Liberals, along with William Gladstone and Thomas Babington Macaulay. Lord Macaulay recorded in his diary: "I have now finished reading again most of Burke's works. Admirable! The greatest man since Milton". The Gladstonian Liberal MP John Morley published two books on Burke (including a biography) and was influenced by Burke, including his views on prejudice. The Cobdenite Radical Francis Hirst thought Burke deserved "a place among English libertarians, even though of all lovers of liberty and of all reformers he was the most conservative, the least abstract, always anxious to preserve and renovate rather than to innovate. In politics he resembled the modern architect who would restore an old house instead of pulling it down to construct a new one on the site". Burke's Reflections on the Revolution in France was controversial at the time of its publication, but after his death, it was to become his best known and most influential work, and a manifesto for Conservative thinking. -The historian Piers Brendon asserts that Burke laid the moral foundations for the British Empire, epitomised in the trial of Warren Hastings, that was ultimately to be its undoing: when Burke stated that "The British Empire must be governed on a plan of freedom, for it will be governed by no other", this was "...an ideological bacillus that would prove fatal. This was Edmund Burke's paternalistic doctrine that colonial government was a trust. It was to be so exercised for the benefit of subject people that they would eventually attain their birthright—freedom". As a consequence of this opinion, Burke objected to the opium trade, which he called a "smuggling adventure" and condemned "the great Disgrace of the British character in India". -Burke's religious writing comprises published works and commentary on the subject of religion. Burke's religious thought was grounded in the belief that religion is the foundation of civil society. He sharply criticised deism and atheism, and emphasised Christianity as a vehicle of social progress. Born in Ireland to a Catholic mother and a Protestant father, Burke vigorously defended the Anglican Church, but also demonstrated sensitivity to Catholic concerns. He linked the conservation of a state (established) religion with the preservation of citizens' constitutional liberties and highlighted Christianity's benefit not only to the believer's soul, but also to political arrangements. diff --git a/e2e/samples/test_documents/Freemasonry.txt b/e2e/samples/test_documents/Freemasonry.txt deleted file mode 100644 index e1671b3648..0000000000 --- a/e2e/samples/test_documents/Freemasonry.txt +++ /dev/null @@ -1,37 +0,0 @@ -Relations between Grand Lodges are determined by the concept of Recognition. Each Grand Lodge maintains a list of other Grand Lodges that it recognises. When two Grand Lodges recognise and are in Masonic communication with each other, they are said to be in amity, and the brethren of each may visit each other's Lodges and interact Masonically. When two Grand Lodges are not in amity, inter-visitation is not allowed. There are many reasons why one Grand Lodge will withhold or withdraw recognition from another, but the two most common are Exclusive Jurisdiction and Regularity. -Since the middle of the 19th century, Masonic historians have sought the origins of the movement in a series of similar documents known as the Old Charges, dating from the Regius Poem in about 1425 to the beginning of the 18th century. Alluding to the membership of a lodge of operative masons, they relate a mythologised history of the craft, the duties of its grades, and the manner in which oaths of fidelity are to be taken on joining. The fifteenth century also sees the first evidence of ceremonial regalia. -A dispute during the Lausanne Congress of Supreme Councils of 1875 prompted the Grand Orient de France to commission a report by a Protestant pastor which concluded that, as Freemasonry was not a religion, it should not require a religious belief. The new constitutions read, "Its principles are absolute liberty of conscience and human solidarity", the existence of God and the immortality of the soul being struck out. It is possible that the immediate objections of the United Grand Lodge of England were at least partly motivated by the political tension between France and Britain at the time. The result was the withdrawal of recognition of the Grand Orient of France by the United Grand Lodge of England, a situation that continues today. -At the dawn of the Grand Lodge era, during the 1720s, James Anderson composed the first printed constitutions for Freemasons, the basis for most subsequent constitutions, which specifically excluded women from Freemasonry. As Freemasonry spread, continental masons began to include their ladies in Lodges of Adoption, which worked three degrees with the same names as the men's but different content. The French officially abandoned the experiment in the early 19th century. Later organisations with a similar aim emerged in the United States, but distinguished the names of the degrees from those of male masonry. -In contrast to Catholic allegations of rationalism and naturalism, Protestant objections are more likely to be based on allegations of mysticism, occultism, and even Satanism. Masonic scholar Albert Pike is often quoted (in some cases misquoted) by Protestant anti-Masons as an authority for the position of Masonry on these issues. However, Pike, although undoubtedly learned, was not a spokesman for Freemasonry and was also controversial among Freemasons in general. His writings represented his personal opinion only, and furthermore an opinion grounded in the attitudes and understandings of late 19th century Southern Freemasonry of the USA. Notably, his book carries in the preface a form of disclaimer from his own Grand Lodge. No one voice has ever spoken for the whole of Freemasonry. -In 1799, English Freemasonry almost came to a halt due to Parliamentary proclamation. In the wake of the French Revolution, the Unlawful Societies Act 1799 banned any meetings of groups that required their members to take an oath or obligation. The Grand Masters of both the Moderns and the Antients Grand Lodges called on Prime Minister William Pitt (who was not a Freemason) and explained to him that Freemasonry was a supporter of the law and lawfully constituted authority and was much involved in charitable work. As a result, Freemasonry was specifically exempted from the terms of the Act, provided that each private lodge's Secretary placed with the local "Clerk of the Peace" a list of the members of his lodge once a year. This continued until 1967 when the obligation of the provision was rescinded by Parliament. -In some countries anti-Masonry is often related to antisemitism and anti-Zionism. For example, In 1980, the Iraqi legal and penal code was changed by Saddam Hussein's ruling Ba'ath Party, making it a felony to "promote or acclaim Zionist principles, including Freemasonry, or who associate [themselves] with Zionist organisations". Professor Andrew Prescott of the University of Sheffield writes: "Since at least the time of the Protocols of the Elders of Zion, antisemitism has gone hand in hand with anti-masonry, so it is not surprising that allegations that 11 September was a Zionist plot have been accompanied by suggestions that the attacks were inspired by a masonic world order". -The bulk of Masonic ritual consists of degree ceremonies. Candidates for Freemasonry are progressively initiated into Freemasonry, first in the degree of Entered Apprentice. Some time later, in a separate ceremony, they will be passed to the degree of Fellowcraft, and finally they will be raised to the degree of Master Mason. In all of these ceremonies, the candidate is entrusted with passwords, signs and grips peculiar to his new rank. Another ceremony is the annual installation of the Master and officers of the Lodge. In some jurisdictions Installed Master is valued as a separate rank, with its own secrets to distinguish its members. In other jurisdictions, the grade is not recognised, and no inner ceremony conveys new secrets during the installation of a new Master of the Lodge. -English Freemasonry spread to France in the 1720s, first as lodges of expatriates and exiled Jacobites, and then as distinctively French lodges which still follow the ritual of the Moderns. From France and England, Freemasonry spread to most of Continental Europe during the course of the 18th century. The Grande Loge de France formed under the Grand Mastership of the Duke of Clermont, who exercised only nominal authority. His successor, the Duke of Orléans, reconstituted the central body as the Grand Orient de France in 1773. Briefly eclipsed during the French Revolution, French Freemasonry continued to grow in the next century. -The majority of Freemasonry considers the Liberal (Continental) strand to be Irregular, and thus withhold recognition. For the Continental lodges, however, having a different approach to Freemasonry was not a reason for severing masonic ties. In 1961, an umbrella organisation, Centre de Liaison et d'Information des Puissances maçonniques Signataires de l'Appel de Strasbourg (CLIPSAS) was set up, which today provides a forum for most of these Grand Lodges and Grand Orients worldwide. Included in the list of over 70 Grand Lodges and Grand Orients are representatives of all three of the above categories, including mixed and women's organisations. The United Grand Lodge of England does not communicate with any of these jurisdictions, and expects its allies to follow suit. This creates the distinction between Anglo-American and Continental Freemasonry. -The denomination with the longest history of objection to Freemasonry is the Roman Catholic Church. The objections raised by the Roman Catholic Church are based on the allegation that Masonry teaches a naturalistic deistic religion which is in conflict with Church doctrine. A number of Papal pronouncements have been issued against Freemasonry. The first was Pope Clement XII's In eminenti apostolatus, 28 April 1738; the most recent was Pope Leo XIII's Ab apostolici, 15 October 1890. The 1917 Code of Canon Law explicitly declared that joining Freemasonry entailed automatic excommunication, and banned books favouring Freemasonry. -In 1933, the Orthodox Church of Greece officially declared that being a Freemason constitutes an act of apostasy and thus, until he repents, the person involved with Freemasonry cannot partake of the Eucharist. This has been generally affirmed throughout the whole Eastern Orthodox Church. The Orthodox critique of Freemasonry agrees with both the Roman Catholic and Protestant versions: "Freemasonry cannot be at all compatible with Christianity as far as it is a secret organisation, acting and teaching in mystery and secret and deifying rationalism." -In addition, most Grand Lodges require the candidate to declare a belief in a Supreme Being. In a few cases, the candidate may be required to be of a specific religion. The form of Freemasonry most common in Scandinavia (known as the Swedish Rite), for example, accepts only Christians. At the other end of the spectrum, "Liberal" or Continental Freemasonry, exemplified by the Grand Orient de France, does not require a declaration of belief in any deity, and accepts atheists (a cause of discord with the rest of Freemasonry). -Exclusive Jurisdiction is a concept whereby only one Grand Lodge will be recognised in any geographical area. If two Grand Lodges claim jurisdiction over the same area, the other Grand Lodges will have to choose between them, and they may not all decide to recognise the same one. (In 1849, for example, the Grand Lodge of New York split into two rival factions, each claiming to be the legitimate Grand Lodge. Other Grand Lodges had to choose between them until the schism was healed.) Exclusive Jurisdiction can be waived when the two over-lapping Grand Lodges are themselves in Amity and agree to share jurisdiction (for example, since the Grand Lodge of Connecticut is in Amity with the Prince Hall Grand Lodge of Connecticut, the principle of Exclusive Jurisdiction does not apply, and other Grand Lodges may recognise both). -There is no clear mechanism by which these local trade organisations became today's Masonic Lodges, but the earliest rituals and passwords known, from operative lodges around the turn of the 17th–18th centuries, show continuity with the rituals developed in the later 18th century by accepted or speculative Masons, as those members who did not practice the physical craft came to be known. The minutes of the Lodge of Edinburgh (Mary's Chapel) No. 1 in Scotland show a continuity from an operative lodge in 1598 to a modern speculative Lodge. It is reputed to be the oldest Masonic Lodge in the world. -Prince Hall Freemasonry exists because of the refusal of early American lodges to admit African-Americans. In 1775, an African-American named Prince Hall, along with fourteen other African-Americans, was initiated into a British military lodge with a warrant from the Grand Lodge of Ireland, having failed to obtain admission from the other lodges in Boston. When the military Lodge left North America, those fifteen men were given the authority to meet as a Lodge, but not to initiate Masons. In 1784, these individuals obtained a Warrant from the Premier Grand Lodge of England (GLE) and formed African Lodge, Number 459. When the UGLE was formed in 1813, all U.S.-based Lodges were stricken from their rolls – due largely to the War of 1812. Thus, separated from both UGLE and any concordantly recognised U.S. Grand Lodge, African Lodge re-titled itself as the African Lodge, Number 1 – and became a de facto "Grand Lodge" (this Lodge is not to be confused with the various Grand Lodges on the Continent of Africa). As with the rest of U.S. Freemasonry, Prince Hall Freemasonry soon grew and organised on a Grand Lodge system for each state. -Maria Deraismes was initiated into Freemasonry in 1882, then resigned to allow her lodge to rejoin their Grand Lodge. Having failed to achieve acceptance from any masonic governing body, she and Georges Martin started a mixed masonic lodge that actually worked masonic ritual. Annie Besant spread the phenomenon to the English speaking world. Disagreements over ritual led to the formation of exclusively female bodies of Freemasons in England, which spread to other countries. Meanwhile, the French had re-invented Adoption as an all-female lodge in 1901, only to cast it aside again in 1935. The lodges, however, continued to meet, which gave rise, in 1959, to a body of women practising continental Freemasonry. -Many Islamic anti-Masonic arguments are closely tied to both antisemitism and Anti-Zionism, though other criticisms are made such as linking Freemasonry to al-Masih ad-Dajjal (the false Messiah). Some Muslim anti-Masons argue that Freemasonry promotes the interests of the Jews around the world and that one of its aims is to destroy the Al-Aqsa Mosque in order to rebuild the Temple of Solomon in Jerusalem. In article 28 of its Covenant, Hamas states that Freemasonry, Rotary, and other similar groups "work in the interest of Zionism and according to its instructions ..." -The preserved records of the Reichssicherheitshauptamt (the Reich Security Main Office) show the persecution of Freemasons during the Holocaust. RSHA Amt VII (Written Records) was overseen by Professor Franz Six and was responsible for "ideological" tasks, by which was meant the creation of antisemitic and anti-Masonic propaganda. While the number is not accurately known, it is estimated that between 80,000 and 200,000 Freemasons were killed under the Nazi regime. Masonic concentration camp inmates were graded as political prisoners and wore an inverted red triangle. -Freemasonry consists of fraternal organisations that trace their origins to the local fraternities of stonemasons, which from the end of the fourteenth century regulated the qualifications of stonemasons and their interaction with authorities and clients. The degrees of freemasonry retain the three grades of medieval craft guilds, those of Apprentice, Journeyman or fellow (now called Fellowcraft), and Master Mason. These are the degrees offered by Craft (or Blue Lodge) Freemasonry. Members of these organisations are known as Freemasons or Masons. There are additional degrees, which vary with locality and jurisdiction, and are usually administered by different bodies than the craft degrees. -Candidates for Freemasonry will have met most active members of the Lodge they are joining before they are initiated. The process varies between jurisdictions, but the candidate will typically have been introduced by a friend at a Lodge social function, or at some form of open evening in the Lodge. In modern times, interested people often track down a local Lodge through the Internet. The onus is on candidates to ask to join; while candidates may be encouraged to ask, they are never invited. Once the initial inquiry is made, an interview usually follows to determine the candidate's suitability. If the candidate decides to proceed from here, the Lodge ballots on the application before he (or she, depending on the Masonic Jurisdiction) can be accepted. -Freemasonry, as it exists in various forms all over the world, has a membership estimated by the United Grand Lodge of England at around six million worldwide. The fraternity is administratively organised into independent Grand Lodges (or sometimes Grand Orients), each of which governs its own Masonic jurisdiction, which consists of subordinate (or constituent) Lodges. The largest single jurisdiction, in terms of membership, is the United Grand Lodge of England (with a membership estimated at around a quarter million). The Grand Lodge of Scotland and Grand Lodge of Ireland (taken together) have approximately 150,000 members. In the United States total membership is just under two million. -The idea of Masonic brotherhood probably descends from a 16th-century legal definition of a brother as one who has taken an oath of mutual support to another. Accordingly, Masons swear at each degree to keep the contents of that degree secret, and to support and protect their brethren unless they have broken the law. In most Lodges the oath or obligation is taken on a Volume of Sacred Law, whichever book of divine revelation is appropriate to the religious beliefs of the individual brother (usually the Bible in the Anglo-American tradition). In Progressive continental Freemasonry, books other than scripture are permissible, a cause of rupture between Grand Lodges. -The earliest known American lodges were in Pennsylvania. The Collector for the port of Pennsylvania, John Moore, wrote of attending lodges there in 1715, two years before the formation of the first Grand Lodge in London. The Premier Grand Lodge of England appointed a Provincial Grand Master for North America in 1731, based in Pennsylvania. Other lodges in the colony obtained authorisations from the later Antient Grand Lodge of England, the Grand Lodge of Scotland, and the Grand Lodge of Ireland, which was particularly well represented in the travelling lodges of the British Army. Many lodges came into existence with no warrant from any Grand Lodge, applying and paying for their authorisation only after they were confident of their own survival. -Masonic lodges existed in Iraq as early as 1917, when the first lodge under the United Grand Lodge of England (UGLE) was opened. Nine lodges under UGLE existed by the 1950s, and a Scottish lodge was formed in 1923. However, the position changed following the revolution, and all lodges were forced to close in 1965. This position was later reinforced under Saddam Hussein; the death penalty was "prescribed" for those who "promote or acclaim Zionist principles, including freemasonry, or who associate [themselves] with Zionist organisations." -The ritual form on which the Grand Orient of France was based was abolished in England in the events leading to the formation of the United Grand Lodge of England in 1813. However the two jurisdictions continued in amity (mutual recognition) until events of the 1860s and 1870s drove a seemingly permanent wedge between them. In 1868 the Supreme Council of the Ancient and Accepted Scottish Rite of the State of Louisiana appeared in the jurisdiction of the Grand Lodge of Louisiana, recognised by the Grand Orient de France, but regarded by the older body as an invasion of their jurisdiction. The new Scottish rite body admitted blacks, and the resolution of the Grand Orient the following year that neither colour, race, nor religion could disqualify a man from Masonry prompted the Grand Lodge to withdraw recognition, and it persuaded other American Grand Lodges to do the same. -In 1983, the Church issued a new code of canon law. Unlike its predecessor, the 1983 Code of Canon Law did not explicitly name Masonic orders among the secret societies it condemns. It states: "A person who joins an association which plots against the Church is to be punished with a just penalty; one who promotes or takes office in such an association is to be punished with an interdict." This named omission of Masonic orders caused both Catholics and Freemasons to believe that the ban on Catholics becoming Freemasons may have been lifted, especially after the perceived liberalisation of Vatican II. However, the matter was clarified when Cardinal Joseph Ratzinger (later Pope Benedict XVI), as the Prefect of the Congregation for the Doctrine of the Faith, issued a Declaration on Masonic Associations, which states: "... the Church's negative judgment in regard to Masonic association remains unchanged since their principles have always been considered irreconcilable with the doctrine of the Church and therefore membership in them remains forbidden. The faithful who enroll in Masonic associations are in a state of grave sin and may not receive Holy Communion." For its part, Freemasonry has never objected to Catholics joining their fraternity. Those Grand Lodges in amity with UGLE deny the Church's claims. The UGLE now states that "Freemasonry does not seek to replace a Mason's religion or provide a substitute for it." -Even in modern democracies, Freemasonry is sometimes viewed with distrust. In the UK, Masons working in the justice system, such as judges and police officers, were from 1999 to 2009 required to disclose their membership. While a parliamentary inquiry found that there has been no evidence of wrongdoing, it was felt that any potential loyalties Masons might have, based on their vows to support fellow Masons, should be transparent to the public. The policy of requiring a declaration of masonic membership of applicants for judicial office (judges and magistrates) was ended in 2009 by Justice Secretary Jack Straw (who had initiated the requirement in the 1990s). Straw stated that the rule was considered disproportionate, since no impropriety or malpractice had been shown as a result of judges being Freemasons. -The Masonic Lodge is the basic organisational unit of Freemasonry. The Lodge meets regularly to conduct the usual formal business of any small organisation (pay bills, organise social and charitable events, elect new members, etc.). In addition to business, the meeting may perform a ceremony to confer a Masonic degree or receive a lecture, which is usually on some aspect of Masonic history or ritual. At the conclusion of the meeting, the Lodge might adjourn for a formal dinner, or festive board, sometimes involving toasting and song. -During the ceremony of initiation, the candidate is expected to swear (usually on a volume of sacred text appropriate to his personal religious faith) to fulfil certain obligations as a Mason. In the course of three degrees, new masons will promise to keep the secrets of their degree from lower degrees and outsiders, and to support a fellow Mason in distress (as far as practicality and the law permit). There is instruction as to the duties of a Freemason, but on the whole, Freemasons are left to explore the craft in the manner they find most satisfying. Some will further explore the ritual and symbolism of the craft, others will focus their involvement on the social side of the Lodge, while still others will concentrate on the charitable functions of the lodge. -Regularity is a concept based on adherence to Masonic Landmarks, the basic membership requirements, tenets and rituals of the craft. Each Grand Lodge sets its own definition of what these landmarks are, and thus what is Regular and what is Irregular (and the definitions do not necessarily agree between Grand Lodges). Essentially, every Grand Lodge will hold that its landmarks (its requirements, tenets and rituals) are Regular, and judge other Grand Lodges based on those. If the differences are significant, one Grand Lodge may declare the other "Irregular" and withdraw or withhold recognition. -All Freemasons begin their journey in the "craft" by being progressively initiated, passed and raised into the three degrees of Craft, or Blue Lodge Masonry. During these three rituals, the candidate is progressively taught the meanings of the Lodge symbols, and entrusted with grips, signs and words to signify to other Masons that he has been so initiated. The initiations are part allegory and part lecture, and revolve around the construction of the Temple of Solomon, and the artistry and death of his chief architect, Hiram Abiff. The degrees are those of Entered apprentice, Fellowcraft and Master Mason. While many different versions of these rituals exist, with at least two different lodge layouts and versions of the Hiram myth, each version is recognisable to any Freemason from any jurisdiction. -The first Grand Lodge, the Grand Lodge of London and Westminster (later called the Grand Lodge of England (GLE)), was founded on 24 June 1717, when four existing London Lodges met for a joint dinner. Many English Lodges joined the new regulatory body, which itself entered a period of self-publicity and expansion. However, many Lodges could not endorse changes which some Lodges of the GLE made to the ritual (they came to be known as the Moderns), and a few of these formed a rival Grand Lodge on 17 July 1751, which they called the "Antient Grand Lodge of England." These two Grand Lodges vied for supremacy until the Moderns promised to return to the ancient ritual. They united on 27 December 1813 to form the United Grand Lodge of England (UGLE). -Widespread segregation in 19th- and early 20th-century North America made it difficult for African-Americans to join Lodges outside of Prince Hall jurisdictions – and impossible for inter-jurisdiction recognition between the parallel U.S. Masonic authorities. By the 1980s, such discrimination was a thing of the past, and today most U.S. Grand Lodges recognise their Prince Hall counterparts, and the authorities of both traditions are working towards full recognition. The United Grand Lodge of England has no problem with recognising Prince Hall Grand Lodges. While celebrating their heritage as lodges of black Americans, Prince Hall is open to all men regardless of race or religion. -In general, Continental Freemasonry is sympathetic to Freemasonry amongst women, dating from the 1890s when French lodges assisted the emergent co-masonic movement by promoting enough of their members to the 33rd degree of the Ancient and Accepted Scottish Rite to allow them, in 1899, to form their own grand council, recognised by the other Continental Grand Councils of that Rite. The United Grand Lodge of England issued a statement in 1999 recognising the two women's grand lodges there to be regular in all but the participants. While they were not, therefore, recognised as regular, they were part of Freemasonry "in general". The attitude of most regular Anglo-American grand lodges remains that women Freemasons are not legitimate Masons. -Since the founding of Freemasonry, many Bishops of the Church of England have been Freemasons, such as Archbishop Geoffrey Fisher. In the past, few members of the Church of England would have seen any incongruity in concurrently adhering to Anglican Christianity and practicing Freemasonry. In recent decades, however, reservations about Freemasonry have increased within Anglicanism, perhaps due to the increasing prominence of the evangelical wing of the church. The former Archbishop of Canterbury, Dr Rowan Williams, appeared to harbour some reservations about Masonic ritual, whilst being anxious to avoid causing offence to Freemasons inside and outside the Church of England. In 2003 he felt it necessary to apologise to British Freemasons after he said that their beliefs were incompatible with Christianity and that he had barred the appointment of Freemasons to senior posts in his diocese when he was Bishop of Monmouth. -In Italy, Freemasonry has become linked to a scandal concerning the Propaganda Due lodge (a.k.a. P2). This lodge was chartered by the Grande Oriente d'Italia in 1877, as a lodge for visiting Masons unable to attend their own lodges. Under Licio Gelli's leadership, in the late 1970s, P2 became involved in the financial scandals that nearly bankrupted the Vatican Bank. However, by this time the lodge was operating independently and irregularly, as the Grand Orient had revoked its charter and expelled Gelli in 1976. diff --git a/e2e/samples/test_documents/John_Kerry.txt b/e2e/samples/test_documents/John_Kerry.txt deleted file mode 100644 index a9a70cc7ee..0000000000 --- a/e2e/samples/test_documents/John_Kerry.txt +++ /dev/null @@ -1,52 +0,0 @@ -Kerry was born in Aurora, Colorado and attended boarding school in Massachusetts and New Hampshire. He graduated from Yale University class of 1966 with a political science major. Kerry enlisted in the Naval Reserve in 1966, and during 1968–1969 served an abbreviated four-month tour of duty in South Vietnam as officer-in-charge (OIC) of a Swift Boat. For that service, he was awarded combat medals that include the Silver Star Medal, Bronze Star Medal, and three Purple Heart Medals. Securing an early return to the United States, Kerry joined the Vietnam Veterans Against the War organization in which he served as a nationally recognized spokesman and as an outspoken opponent of the Vietnam War. He appeared in the Fulbright Hearings before the Senate Committee on Foreign Affairs where he deemed United States war policy in Vietnam to be the cause of war crimes. -After receiving his J.D. from Boston College Law School, Kerry worked in Massachusetts as an Assistant District Attorney. He served as Lieutenant Governor of Massachusetts under Michael Dukakis from 1983 to 1985 and was elected to the U.S. Senate in 1984 and was sworn in the following January. On the Senate Foreign Relations Committee, he led a series of hearings from 1987 to 1989 which were a precursor to the Iran–Contra affair. Kerry was re-elected to additional terms in 1990, 1996, 2002 and 2008. In 2002, Kerry voted to authorize the President "to use force, if necessary, to disarm Saddam Hussein", but warned that the administration should exhaust its diplomatic avenues before launching war. -In his 2004 presidential campaign, Kerry criticized George W. Bush for the Iraq War. He and his running mate, North Carolina Senator John Edwards, lost the election, finishing 35 electoral votes behind Bush and Vice President Dick Cheney. Kerry returned to the Senate, becoming Chairman of the Senate Committee on Small Business and Entrepreneurship in 2007 and then of the Foreign Relations Committee in 2009. In January 2013, Kerry was nominated by President Barack Obama to succeed outgoing Secretary of State Hillary Clinton and then confirmed by the U.S. Senate, assuming the office on February 1, 2013. -John Forbes Kerry was born on December 11, 1943 in Aurora, Colorado, at Fitzsimons Army Hospital. He was the second oldest of four children born to Richard John Kerry, a Foreign Service officer and lawyer, and Rosemary Isabel Forbes, a nurse and social activist. His father was raised Catholic (John's paternal grandparents were Austro-Hungarian Jewish immigrants who converted to Catholicism) and his mother was Episcopalian. He was raised with an elder sister named Margaret (born 1941), a younger sister named Diana (born 1947) and a younger brother named Cameron (born 1950). The children were raised in their father's faith; John Kerry served as an altar boy. -In his sophomore year, Kerry became the Chairman of the Liberal Party of the Yale Political Union, and a year later he served as President of the Union. Amongst his influential teachers in this period was Professor H. Bradford Westerfield, who was himself a former President of the Political Union. His involvement with the Political Union gave him an opportunity to be involved with important issues of the day, such as the civil rights movement and the New Frontier program. He also became a member of the secretive Skull and Bones Society, and traveled to Switzerland through AIESEC Yale. -On February 18, 1966, Kerry enlisted in the Naval Reserve. He began his active duty military service on August 19, 1966. After completing 16 weeks of Officer Candidate School at the U.S. Naval Training Center in Newport, Rhode Island, Kerry received his officer's commission on December 16, 1966. During the 2004 election, Kerry posted his military records at his website, and permitted reporters to inspect his medical records. In 2005, Kerry released his military and medical records to the representatives of three news organizations, but has not authorized full public access to those records. -During his tour on the guided missile frigate USS Gridley, Kerry requested duty in South Vietnam, listing as his first preference a position as the commander of a Fast Patrol Craft (PCF), also known as a "Swift boat." These 50-foot (15 m) boats have aluminum hulls and have little or no armor, but are heavily armed and rely on speed. "I didn't really want to get involved in the war", Kerry said in a book of Vietnam reminiscences published in 1986. "When I signed up for the swift boats, they had very little to do with the war. They were engaged in coastal patrolling and that's what I thought I was going to be doing." However, his second choice of billet was on a river patrol boat, or "PBR", which at the time was serving a more dangerous duty on the rivers of Vietnam. -During the night of December 2 and early morning of December 3, 1968, Kerry was in charge of a small boat operating near a peninsula north of Cam Ranh Bay together with a Swift boat (PCF-60). According to Kerry and the two crewmen who accompanied him that night, Patrick Runyon and William Zaladonis, they surprised a group of Vietnamese men unloading sampans at a river crossing, who began running and failed to obey an order to stop. As the men fled, Kerry and his crew opened fire on the sampans and destroyed them, then rapidly left. During this encounter, Kerry received a shrapnel wound in the left arm above the elbow. It was for this injury that Kerry received his first Purple Heart Medal. -Kerry received his second Purple Heart for a wound received in action on the Bồ Đề River on February 20, 1969. The plan had been for the Swift boats to be accompanied by support helicopters. On the way up the Bo De, however, the helicopters were attacked. As the Swift boats reached the Cửa Lớn River, Kerry's boat was hit by a B-40 rocket (rocket propelled grenade round), and a piece of shrapnel hit Kerry's left leg, wounding him. Thereafter, enemy fire ceased and his boat reached the Gulf of Thailand safely. Kerry continues to have shrapnel embedded in his left thigh because the doctors that first treated him decided to remove the damaged tissue and close the wound with sutures rather than make a wide opening to remove the shrapnel. Though wounded like several others earlier that day, Kerry did not lose any time off from duty. -Eight days later, on February 28, 1969, came the events for which Kerry was awarded his Silver Star Medal. On this occasion, Kerry was in tactical command of his Swift boat and two other Swift boats during a combat operation. Their mission on the Duong Keo River included bringing an underwater demolition team and dozens of South Vietnamese Marines to destroy enemy sampans, structures and bunkers as described in the story The Death Of PCF 43. Running into heavy small arms fire from the river banks, Kerry "directed the units to turn to the beach and charge the Viet Cong positions" and he "expertly directed" his boat's fire causing the enemy to flee while at the same time coordinating the insertion of the ninety South Vietnamese troops (according to the original medal citation signed by Admiral Zumwalt). Moving a short distance upstream, Kerry's boat was the target of a B-40 rocket round; Kerry charged the enemy positions and as his boat hove to and beached, a Viet Cong ("VC") insurgent armed with a rocket launcher emerged from a spider hole and ran. While the boat's gunner opened fire, wounding the VC in the leg, and while the other boats approached and offered cover fire, Kerry jumped from the boat to pursue the VC insurgent, subsequently killing him and capturing his loaded rocket launcher. -Kerry's commanding officer, Lieutenant Commander George Elliott, stated to Douglas Brinkley in 2003 that he did not know whether to court-martial Kerry for beaching the boat without orders or give him a medal for saving the crew. Elliott recommended Kerry for the Silver Star, and Zumwalt flew into An Thoi to personally award medals to Kerry and the rest of the sailors involved in the mission. The Navy's account of Kerry's actions is presented in the original medal citation signed by Zumwalt. The engagement was documented in an after-action report, a press release written on March 1, 1969, and a historical summary dated March 17, 1969. -On March 13, 1969, on the Bái Háp River, Kerry was in charge of one of five Swift boats that were returning to their base after performing an Operation Sealords mission to transport South Vietnamese troops from the garrison at Cái Nước and MIKE Force advisors for a raid on a Vietcong camp located on the Rach Dong Cung canal. Earlier in the day, Kerry received a slight shrapnel wound in the buttocks from blowing up a rice bunker. Debarking some but not all of the passengers at a small village, the boats approached a fishing weir; one group of boats went around to the left of the weir, hugging the shore, and a group with Kerry's PCF-94 boat went around to the right, along the shoreline. A mine was detonated directly beneath the lead boat, PCF-3, as it crossed the weir to the left, lifting PCF-3 "about 2-3 ft out of water". -James Rassmann, a Green Beret advisor who was aboard Kerry's PCF-94, was knocked overboard when, according to witnesses and the documentation of the event, a mine or rocket exploded close to the boat. According to the documentation for the event, Kerry's arm was injured when he was thrown against a bulkhead during the explosion. PCF 94 returned to the scene and Kerry rescued Rassmann who was receiving sniper fire from the water. Kerry received the Bronze Star Medal with Combat "V" for "heroic achievement", for his actions during this incident; he also received his third Purple Heart. -After Kerry's third qualifying wound, he was entitled per Navy regulations to reassignment away from combat duties. Kerry's preferred choice for reassignment was as a military aide in Boston, New York or Washington, D.C. On April 11, 1969, he reported to the Brooklyn-based Atlantic Military Sea Transportation Service, where he would remain on active duty for the following year as a personal aide to an officer, Rear Admiral Walter Schlech. On January 1, 1970 Kerry was temporarily promoted to full Lieutenant. Kerry had agreed to an extension of his active duty obligation from December 1969 to August 1970 in order to perform Swift Boat duty. John Kerry was on active duty in the United States Navy from August 1966 until January 1970. He continued to serve in the Naval Reserve until February 1978. -With the continuing controversy that had surrounded the military service of George W. Bush since the 2000 Presidential election (when he was accused of having used his father's political influence to gain entrance to the Texas Air National Guard, thereby protecting himself from conscription into the United States Army, and possible service in the Vietnam War), John Kerry's contrasting status as a decorated Vietnam War veteran posed a problem for Bush's re-election campaign, which Republicans sought to counter by calling Kerry's war record into question. As the presidential campaign of 2004 developed, approximately 250 members of a group called Swift Boat Veterans for Truth (SBVT, later renamed Swift Vets and POWs for Truth) opposed Kerry's campaign. The group held press conferences, ran ads and endorsed a book questioning Kerry's service record and his military awards. The group included several members of Kerry's unit, such as Larry Thurlow, who commanded a swift boat alongside of Kerry's, and Stephen Gardner, who served on Kerry's boat. The campaign inspired the widely used political pejorative 'swiftboating', to describe an unfair or untrue political attack. Most of Kerry's former crewmates have stated that SBVT's allegations are false. -After returning to the United States, Kerry joined the Vietnam Veterans Against the War (VVAW). Then numbering about 20,000, VVAW was considered by some (including the administration of President Richard Nixon) to be an effective, if controversial, component of the antiwar movement. Kerry participated in the "Winter Soldier Investigation" conducted by VVAW of U.S. atrocities in Vietnam, and he appears in a film by that name that documents the investigation. According to Nixon Secretary of Defense Melvin Laird, "I didn't approve of what he did, but I understood the protesters quite well", and he declined two requests from the Navy to court martial Reserve Lieutenant Kerry over his antiwar activity. -On April 22, 1971, Kerry appeared before a U.S. Senate committee hearing on proposals relating to ending the war. The day after this testimony, Kerry participated in a demonstration with thousands of other veterans in which he and other Vietnam War veterans threw their medals and service ribbons over a fence erected at the front steps of the United States Capitol building to dramatize their opposition to the war. Jack Smith, a Marine, read a statement explaining why the veterans were returning their military awards to the government. For more than two hours, almost 1000 angry veterans tossed their medals, ribbons, hats, jackets, and military papers over the fence. Each veteran gave his or her name, hometown, branch of service and a statement. Kerry threw some of his own decorations and awards as well as some given to him by other veterans to throw. As Kerry threw his decorations over the fence, his statement was: "I'm not doing this for any violent reasons, but for peace and justice, and to try and make this country wake up once and for all." -Kerry was arrested on May 30, 1971, during a VVAW march to honor American POWs held captive by North Vietnam. The march was planned as a multi-day event from Concord to Boston, and while in Lexington, participants tried to camp on the village green. At 2:30 a.m., local and state police arrested 441 demonstrators, including Kerry, for trespassing. All were given the Miranda Warning and were hauled away on school buses to spend the night at the Lexington Public Works Garage. Kerry and the other protesters later paid a $5 fine, and were released. The mass arrests caused a community backlash and ended up giving positive coverage to the VVAW. -In 1970, Kerry had considered running for Congress in the Democratic primary against hawkish Democrat Philip J. Philbin of Massachusetts's 3rd congressional district, but deferred in favour of Robert Drinan, a Jesuit priest and anti-war activist, who went on to defeat Philbin. In February 1972, Kerry's wife bought a house in Worcester, with Kerry intending to run against the 4th district's ageing thirteen-term incumbent Democrat, Harold Donohue. The couple never moved in. After Republican Congressman F. Bradford Morse of the neighbouring 5th district announced his retirement and then resignation to become Under-Secretary-General for Political and General Assembly Affairs at the United Nations. The couple instead rented an apartment in Lowell, so that Kerry could run to succeed him. -Including Kerry, the Democratic primary race had 10 candidates, including attorney Paul J. Sheehy, State Representative Anthony R. DiFruscia, John J. Desmond and Robert B. Kennedy. Kerry ran a "very expensive, sophisticated campaign", financed by out-of-state backers and supported by many young volunteers. DiFruscia's campaign headquarters shared the same building as Kerry's. On the eve of the September 19 primary, police found Kerry's younger brother Cameron and campaign field director Thomas J. Vallely, breaking into where the building's telephone lines were located. They were arrested and charged with "breaking and entering with the intent to commit grand larceny", but the charges were dropped a year later. At the time of the incident, DiFruscia alleged that the two were trying to disrupt his get-out-the vote efforts. Vallely and Cameron Kerry maintained that they were only checking their own telephone lines because they had received an anonymous call warning that the Kerry lines would be cut. -In the general election, Kerry was initially favored to defeat the Republican candidate, former State Representative Paul W. Cronin, and conservative Democrat Roger P. Durkin, who ran as an Independent. A week after the primary, one poll put Kerry 26-points ahead of Cronin. His campaign called for a national health insurance system, discounted prescription drugs for the unemployed, a jobs programme to clean up the Merrimack River and rent controls in Lowell and Lawrence. A major obstacle, however, was the district's leading newspaper, the conservative The Sun. The paper editorialized against him. It also ran critical news stories about his out-of-state contributions and his "carpetbagging", because he had only moved into the district in April. Subsequently released "Watergate" Oval Office tape recordings of the Nixon White House showed that defeating Kerry's candidacy had attracted the personal attention of President Nixon. Kerry himself asserts that Nixon sent operatives to Lowell to help derail his campaign. -The race was the most expensive for Congress in the country that year and four days before the general election, Durkin withdrew and endorsed Cronin, hoping to see Kerry defeated. The week before, a poll had put Kerry 10 points ahead of Cronin, with Dukin on 13%. In the final days of the campaign, Kerry sensed that it was "slipping away" and Cronin emerged victorious by 110,970 votes (53.45%) to Kerry's 92,847 (44.72%). After his defeat, Kerry lamented in a letter to supporters that "for two solid weeks, [The Sun] called me un-American, New Left antiwar agitator, unpatriotic, and labeled me every other 'un-' and 'anti-' that they could find. It's hard to believe that one newspaper could be so powerful, but they were." He later felt that his failure to respond directly to The Sun's attacks cost him the race. -After Kerry's 1972 defeat, he and his wife bought a house in Belvidere, Lowell, entering a decade which his brother Cameron later called "the years in exile". He spent some time working as a fundraiser for the Cooperative for Assistance and Relief Everywhere (CARE), an international humanitarian organization. In September 1973, he entered Boston College Law School. While studying, Kerry worked as a talk radio host on WBZ and, in July 1974, was named executive director of Mass Action, a Massachusetts advocacy association. -In January 1977, Droney promoted him to First Assistant District Attorney, essentially making Kerry his campaign and media surrogate because Droney was afflicted with amyotrophic lateral sclerosis (ALS, or Lou Gehrig's Disease). As First Assistant, Kerry tried cases, which included winning convictions in a high-profile rape case and a murder. He also played a role in administering the office, including initiating the creation of special white-collar and organized crime units, creating programs to address the problems of rape and other crime victims and witnesses, and managing trial calendars to reflect case priorities. It was in this role in 1978 that Kerry announced an investigation into possible criminal charges against then Senator Edward Brooke, regarding "misstatements" in his first divorce trial. The inquiry ended with no charges being brought after investigators and prosecutors determined that Brooke's misstatements were pertinent to the case, but were not material enough to have affected the outcome. -Droney's health was poor and Kerry had decided to run for his position in the 1978 election should Droney drop out. However, Droney was re-elected and his health improved; he went on to re-assume many of the duties that he had delegated to Kerry. Kerry thus decided to leave, departing in 1979 with assistant DA Roanne Sragow to set up their own law firm. Kerry also worked as a commentator for WCVB-TV and co-founded a bakery, Kilvert & Forbes Ltd., with businessman and former Kennedy aide K. Dun Gifford. -The junior U.S. Senator from Massachusetts, Paul Tsongas, announced in 1984 that he would be stepping down for health reasons. Kerry ran, and as in his 1982 race for Lieutenant Governor, he did not receive the endorsement of the party regulars at the state Democratic convention. Congressman James Shannon, a favorite of House Speaker Tip O'Neill, was the early favorite to win the nomination, and he "won broad establishment support and led in early polling." Again as in 1982, however, Kerry prevailed in a close primary. -On April 18, 1985, a few months after taking his Senate seat, Kerry and Senator Tom Harkin of Iowa traveled to Nicaragua and met the country's president, Daniel Ortega. Though Ortega had won internationally certified elections, the trip was criticized because Ortega and his leftist Sandinista government had strong ties to Cuba and the USSR and were accused of human rights abuses. The Sandinista government was opposed by the right-wing CIA-backed rebels known as the Contras. While in Nicaragua, Kerry and Harkin talked to people on both sides of the conflict. Through the senators, Ortega offered a cease-fire agreement in exchange for the U.S. dropping support of the Contras. The offer was denounced by the Reagan administration as a "propaganda initiative" designed to influence a House vote on a $14 million Contra aid package, but Kerry said "I am willing..... to take the risk in the effort to put to test the good faith of the Sandinistas." The House voted down the Contra aid, but Ortega flew to Moscow to accept a $200 million loan the next day, which in part prompted the House to pass a larger $27 million aid package six weeks later. -Meanwhile, Kerry's staff began their own investigations and, on October 14, issued a report that exposed illegal activities on the part of Lieutenant Colonel Oliver North, who had set up a private network involving the National Security Council and the CIA to deliver military equipment to right-wing Nicaraguan rebels (Contras). In effect, North and certain members of the President's administration were accused by Kerry's report of illegally funding and supplying armed militants without the authorization of Congress. Kerry's staff investigation, based on a year-long inquiry and interviews with fifty unnamed sources, is said to raise "serious questions about whether the United States has abided by the law in its handling of the contras over the past three years." -The Kerry Committee report found that "the Contra drug links included..... payments to drug traffickers by the U.S. State Department of funds authorized by the Congress for humanitarian assistance to the Contras, in some cases after the traffickers had been indicted by federal law enforcement agencies on drug charges, in others while traffickers were under active investigation by these same agencies." The U.S. State Department paid over $806,000 to known drug traffickers to carry humanitarian assistance to the Contras. Kerry's findings provoked little reaction in the media and official Washington. -During their investigation of Noriega, Kerry's staff found reason to believe that the Pakistan-based Bank of Credit and Commerce International (BCCI) had facilitated Noriega's drug trafficking and money laundering. This led to a separate inquiry into BCCI, and as a result, banking regulators shut down BCCI in 1991. In December 1992, Kerry and Senator Hank Brown, a Republican from Colorado, released The BCCI Affair, a report on the BCCI scandal. The report showed that the bank was crooked and was working with terrorists, including Abu Nidal. It blasted the Department of Justice, the Department of the Treasury, the Customs Service, the Federal Reserve Bank, as well as influential lobbyists and the CIA. -In 1996, Kerry faced a difficult re-election fight against Governor William Weld, a popular Republican incumbent who had been re-elected in 1994 with 71% of the vote. The race was covered nationwide as one of the most closely watched Senate races that year. Kerry and Weld held several debates and negotiated a campaign spending cap of $6.9 million at Kerry's Beacon Hill townhouse. Both candidates spent more than the cap, with each camp accusing the other of being first to break the agreement. During the campaign, Kerry spoke briefly at the 1996 Democratic National Convention. Kerry won re-election with 53 percent to Weld's 45 percent. -Kerry said that he had intended the remark as a jab at President Bush, and described the remarks as a "botched joke", having inadvertently left out the key word "us" (which would have been, "If you don't, you get us stuck in Iraq"), as well as leaving the phrase "just ask President Bush" off of the end of the sentence. In Kerry's prepared remarks, which he released during the ensuing media frenzy, the corresponding line was "... you end up getting us stuck in a war in Iraq. Just ask President Bush." He also said that from the context of the speech which, prior to the "stuck in Iraq" line, made several specific references to Bush and elements of his biography, that Kerry was referring to President Bush and not American troops in general. -Kerry "has emerged in the past few years as an important envoy for Afghanistan and Pakistan during times of crisis," a Washington Post report stated in May 2011, as Kerry undertook another trip to the two countries. The killing of Osama bin Laden "has generated perhaps the most important crossroads yet," the report continued, as the senator spoke at a press conference and prepared to fly from Kabul to Pakistan. Among matters discussed during the May visit to Pakistan, under the general rubric of "recalibrating" the bilateral relationship, Kerry sought and retrieved from the Pakistanis the tail-section of the U.S. helicopter which had had to be abandoned at Abbottabad during the bin Laden strike. In 2013, Kerry met with Pakistan's army chief Gen. Ashfaq Parvez Kayani to discuss the peace process with the Taliban in Afghanistan. -Most analyses place Kerry's voting record on the left within the Senate Democratic caucus. During the 2004 presidential election he was portrayed as a staunch liberal by conservative groups and the Bush campaign, who often noted that in 2003 Kerry was rated the National Journal's top Senate liberal. However, that rating was based only upon voting on legislation within that past year. In fact, in terms of career voting records, the National Journal found that Kerry is the 11th most liberal member of the Senate. Most analyses find that Kerry is at least slightly more liberal than the typical Democratic Senator. Kerry has stated that he opposes privatizing Social Security, supports abortion rights for adult women and minors, supports same-sex marriage, opposes capital punishment except for terrorists, supports most gun control laws, and is generally a supporter of trade agreements. Kerry supported the North American Free Trade Agreement and Most Favored Nation status for China, but opposed the Central American Free Trade Agreement.[citation needed] -In the lead up to the Iraq War, Kerry said on October 9, 2002; "I will be voting to give the President of the United States the authority to use force, if necessary, to disarm Saddam Hussein because I believe that a deadly arsenal of weapons of mass destruction in his hands is a real and grave threat to our security." Bush relied on that resolution in ordering the 2003 invasion of Iraq. Kerry also gave a January 23, 2003 speech to Georgetown University saying "Without question, we need to disarm Saddam Hussein. He is a brutal, murderous dictator; leading an oppressive regime he presents a particularly grievous threat because he is so consistently prone to miscalculation. So the threat of Saddam Hussein with weapons of mass destruction is real." Kerry did, however, warn that the administration should exhaust its diplomatic avenues before launching war: "Mr. President, do not rush to war, take the time to build the coalition, because it's not winning the war that's hard, it's winning the peace that's hard." -Kerry chaired the Senate Select Committee on POW/MIA Affairs from 1991 to 1993. The committee's report, which Kerry endorsed, stated there was "no compelling evidence that proves that any American remains alive in captivity in Southeast Asia." In 1994 the Senate passed a resolution, sponsored by Kerry and fellow Vietnam veteran John McCain, that called for an end to the existing trade embargo against Vietnam; it was intended to pave the way for normalization. In 1995, President Bill Clinton normalized diplomatic relations with the country of Vietnam. -In the 2004 Democratic presidential primaries, John Kerry defeated several Democratic rivals, including Sen. John Edwards (D-North Carolina.), former Vermont Governor Howard Dean and retired Army General Wesley Clark. His victory in the Iowa caucuses is widely believed to be the tipping point where Kerry revived his sagging campaign in New Hampshire and the February 3, 2004, primary states like Arizona, South Carolina and New Mexico. Kerry then went on to win landslide victories in Nevada and Wisconsin. Kerry thus won the Democratic nomination to run for President of the United States against incumbent George W. Bush. On July 6, 2004, he announced his selection of John Edwards as his running mate. Democratic strategist Bob Shrum, who was Kerry's 2004 campaign adviser, wrote an article in Time magazine claiming that after the election, Kerry had said that he wished he'd never picked Edwards, and that the two have since stopped speaking to each other. In a subsequent appearance on ABC's This Week, Kerry refused to respond to Shrum's allegation, calling it a "ridiculous waste of time." -During his bid to be elected president in 2004, Kerry frequently criticized President George W. Bush for the Iraq War. While Kerry had initially voted in support of authorizing President Bush to use force in dealing with Saddam Hussein, he voted against an $87 billion supplemental appropriations bill to pay for the subsequent war. His statement on March 16, 2004, "I actually did vote for the $87 billion before I voted against it," helped the Bush campaign to paint him as a flip-flopper and has been cited as contributing to Kerry's defeat. -Kerry established a separate political action committee, Keeping America's Promise, which declared as its mandate "A Democratic Congress will restore accountability to Washington and help change a disastrous course in Iraq", and raised money and channeled contributions to Democratic candidates in state and federal races. Through Keeping America's Promise in 2005, Kerry raised over $5.5 million for other Democrats up and down the ballot. Through his campaign account and his political action committee, the Kerry campaign operation generated more than $10 million for various party committees and 179 candidates for the U.S. House, Senate, state and local offices in 42 states focusing on the midterm elections during the 2006 election cycle. "Cumulatively, John Kerry has done as much if not more than any other individual senator", Hassan Nemazee, the national finance chairman of the DSCC said. -On December 15, 2012, several news outlets reported that President Barack Obama would nominate Kerry to succeed Hillary Clinton as Secretary of State, after Susan Rice, widely seen as Obama's preferred choice, withdrew her name from consideration citing a politicized confirmation process following criticism of her response to the 2012 Benghazi attack. On December 21, Obama proposed the nomination which received positive commentary. His confirmation hearing took place on January 24, 2013, before the Senate Foreign Relations Committee, the same panel where he first testified in 1971. The committee unanimously voted to approve him on January 29, 2013, and the same day the full Senate confirmed him on a vote of 94–3. In a letter to Massachusetts Governor Deval Patrick, Kerry announced his resignation from the Senate effective February 1. -In the State Department, Kerry quickly earned a reputation "for being aloof, keeping to himself, and not bothering to read staff memos." Career State Department officials have complained that power has become too centralized under Kerry's leadership, which slows department operations when Kerry is on one of his frequent overseas trips. Others in State describe Kerry as having "a kind of diplomatic attention deficit disorder" as he shifts from topic to topic instead of focusing on long-term strategy. When asked whether he was traveling too much, he responded, "Hell no. I'm not slowing down." Despite Kerry's early achievements, morale at State is lower than under Hillary Clinton according to department employees. However, after Kerry's first six months in the State Department, a Gallup poll found he had high approval ratings among Americans as Secretary of State. After a year, another poll showed Kerry's favorability continued to rise. Less than two years into Kerry's term, the Foreign Policy Magazine's 2014 Ivory Tower survey of international relations scholars asked, "Who was the most effective U.S. Secretary of State in the past 50 years?"; John Kerry and Lawrence Eagleburger tied for 11th place out of the 15 confirmed Secretaries of State in that period. -In January 2014, having met with Secretary of State, Archbishop Pietro Parolin, Kerry said "We touched on just about every major issue that we are both working on, that are issues of concern to all of us. First of all, we talked at great length about Syria, and I was particularly appreciative for the Archbishop’s raising this issue, and equally grateful for the Holy Father’s comments – the Pope's comments yesterday regarding his support for the Geneva II process. We welcome that support. It is very important to have broad support, and I know that the Pope is particularly concerned about the massive numbers of displaced human beings and the violence that has taken over 130,000 lives". -Kerry said on September 9 in response to a reporter's question about whether Syrian President Bashar al-Assad could avert a military strike: "He could turn over every single bit of his chemical weapons to the international community in the next week. Turn it over, all of it, without delay, and allow a full and total accounting for that. But he isn't about to do it, and it can't be done, obviously." This unscripted remark initiated a process that would lead to Syria agreeing to relinquish and destroy its chemical weapons arsenal, as Russia treated Kerry's statement as a serious proposal. Russian Foreign Minister Sergey Lavrov said Russia would work "immediately" to convince Syria relinquish and destroy its large chemical weapons arsenal. Syria quickly welcomed this proposal and on September 14, the UN formally accepted Syria's application to join the convention banning chemical weapons, and separately, the U.S. and Russia agreed on a plan to eliminate Syria's chemical weapons by the middle of 2014. On September 28, the UN Security Council passed a resolution ordering the destruction of Syria's chemical weapons and condemning the August 21 Ghouta attack. -In a speech before the Organization of American States in November 2013, Kerry remarked that the era of the Monroe Doctrine was over. He went on to explain, "The relationship that we seek and that we have worked hard to foster is not about a United States declaration about how and when it will intervene in the affairs of other American states. It's about all of our countries viewing one another as equals, sharing responsibilities, cooperating on security issues, and adhering not to doctrine, but to the decisions that we make as partners to advance the values and the interests that we share." -Kerry's paternal grandparents, shoe businessman Frederick A. "Fred" Kerry and musician Ida Lowe, were immigrants from the Austro-Hungarian Empire. Fred was born as "Fritz Kohn" before he and Ida took on the "Kerry" name and moved to the United States. Fred and Ida were born Jewish, and converted to Catholicism together in Austria. His maternal ancestors were of Scottish and English descent, and his maternal grandfather James Grant Forbes II was a member of the Forbes family, while his maternal grandmother Margaret Tyndal Winthrop was a member of the Dudley–Winthrop family. Margaret's paternal grandfather Robert Charles Winthrop served as the 22nd Speaker of the U.S. House of Representatives. Robert's father was Governor Thomas Lindall Winthrop. Thomas' father John Still Winthrop was a great-great-grandson of Massachusetts Bay Colony Governor John Winthrop and great-grandson of Governor Thomas Dudley. Through his mother, John is a first cousin once removed of French politician Brice Lalonde. -Alexandra was born days before Kerry began law school. In 1982, Julia asked Kerry for a separation while she was suffering from severe depression. They were divorced on July 25, 1988, and the marriage was formally annulled in 1997. "After 14 years as a political wife, I associated politics only with anger, fear and loneliness" she wrote in A Change of Heart, her book about depression. Thorne later married Richard Charlesworth, an architect, and moved to Bozeman, Montana, where she became active in local environmental groups such as the Greater Yellowstone Coalition. Thorne supported Kerry's 2004 presidential run. She died of cancer on April 27, 2006. -Kerry and his second wife, Mozambican-born businesswoman and philanthropist Maria Teresa Thierstein Simões Ferreira (known as Teresa), the widow of Kerry's late Pennsylvania Republican Senate colleague Henry John Heinz III, were introduced to each other by Heinz at an Earth Day rally in 1990. Early the following year, Senator Heinz was killed in a plane crash near Lower Merion. Teresa has three sons from her previous marriage to Heinz, Henry John Heinz IV, André Thierstein Heinz, and Christopher Drake Heinz. Heinz and Kerry were married on May 26, 1995, in Nantucket, Massachusetts. -The Forbes 400 survey estimated in 2004 that Teresa Heinz Kerry had a net worth of $750 million. However, estimates have frequently varied, ranging from around $165 million to as high as $3.2 billion, according to a study in the Los Angeles Times. Regardless of which figure is correct, Kerry was the wealthiest U.S. Senator while serving in the Senate. Independent of Heinz, Kerry is wealthy in his own right, and is the beneficiary of at least four trusts inherited from Forbes family relatives, including his mother, Rosemary Forbes Kerry, who died in 2002. Forbes magazine (named for the Forbes family of publishers, unrelated to Kerry) estimated that if elected, and if Heinz family assets were included, Kerry would have been the third-richest U.S. President in history, when adjusted for inflation. This assessment was based on Heinz and Kerry's combined assets, but the couple signed a prenuptial agreement that keeps their assets separate. Kerry's financial disclosure form for 2011 put his personal assets in the range of $230,000,000 to $320,000,000, including the assets of his spouse and any dependent children. This included slightly more than three million dollars worth of H. J. Heinz Company assets, which increased in value by over six hundred thousand dollars in 2013 when Berkshire Hathaway announced their intention to purchase the company. -Kerry is a Roman Catholic, and is said to carry a religious rosary, a prayer book, and a St. Christopher medal (the patron saint of travelers) when he campaigned. While Kerry is personally against abortion, he supports a woman's legal right to have one. Discussing his faith, Kerry said, "I thought of being a priest. I was very religious while at school in Switzerland. I was an altar boy and prayed all the time. I was very centered around the Mass and the church." He also said that the Letters of Paul (Apostle Paul) moved him the most, stating that they taught him to "not feel sorry for myself." -Kerry told Christianity Today in October 2004 "I'm a Catholic and I practice, but at the same time I have an open-mindedness to many other expressions of spirituality that come through different religions... I've spent some time reading and thinking about religion and trying to study it, and I've arrived at not so much a sense of the differences, but a sense of the similarities in so many ways." He said that he believed that the Torah, the Qur'an, and the Bible all share a fundamental story which connects with readers. -In addition to the sports he played at Yale, Kerry is described by Sports Illustrated, among others, as an "avid cyclist", primarily riding on a road bike. Prior to his presidential bid, Kerry was known to have participated in several long-distance rides (centuries). Even during his many campaigns, he was reported to have visited bicycle stores in both his home state and elsewhere. His staff requested recumbent stationary bikes for his hotel rooms. He has also been a snowboarder, windsurfer, and sailor. -According to the Boston Herald, dated July 23, 2010, Kerry commissioned construction on a new $7 million yacht (a Friendship 75) in New Zealand and moored it in Portsmouth, Rhode Island, where the Friendship yacht company is based. The article claimed this allowed him to avoid paying Massachusetts taxes on the property including approximately $437,500 in sales tax and an annual excise tax of about $500. However, on July 27, 2010, Kerry stated he had yet to take legal possession of the boat, had not intended to avoid the taxes, and that when he took possession, he would pay the taxes whether he owed them or not. diff --git a/e2e/samples/test_documents/Late_Middle_Ages.txt b/e2e/samples/test_documents/Late_Middle_Ages.txt deleted file mode 100644 index c5fc31f914..0000000000 --- a/e2e/samples/test_documents/Late_Middle_Ages.txt +++ /dev/null @@ -1,40 +0,0 @@ -Around 1300, centuries of prosperity and growth in Europe came to a halt. A series of famines and plagues, including the Great Famine of 1315–1317 and the Black Death, reduced the population to around half of what it was before the calamities. Along with depopulation came social unrest and endemic warfare. France and England experienced serious peasant uprisings, such as the Jacquerie and the Peasants' Revolt, as well as over a century of intermittent conflict in the Hundred Years' War. To add to the many problems of the period, the unity of the Catholic Church was shattered by the Western Schism. Collectively these events are sometimes called the Crisis of the Late Middle Ages. -Despite these crises, the 14th century was also a time of great progress in the arts and sciences. Following a renewed interest in ancient Greek and Roman texts that took root in the High Middle Ages, the Italian Renaissance began. The absorption of Latin texts had started before the Renaissance of the 12th century through contact with Arabs during the Crusades, but the availability of important Greek texts accelerated with the capture of Constantinople by the Ottoman Turks, when many Byzantine scholars had to seek refuge in the West, particularly Italy. -Combined with this influx of classical ideas was the invention of printing which facilitated dissemination of the printed word and democratized learning. These two things would later lead to the Protestant Reformation. Toward the end of the period, an era of discovery began (Age of Discovery). The rise of the Ottoman Empire, culminating in the Fall of Constantinople in 1453, eroded the last remnants of the Byzantine Empire and cut off trading possibilities with the east. Europeans were forced to seek new trading routes, leading to the expedition of Columbus to the Americas in 1492, and Vasco da Gama’s circumnavigation of India and Africa in 1498. Their discoveries strengthened the economy and power of European nations. -The term "Late Middle Ages" refers to one of the three periods of the Middle Ages, along with the Early Middle Ages and the High Middle Ages. Leonardo Bruni was the first historian to use tripartite periodization in his History of the Florentine People (1442). Flavio Biondo used a similar framework in Decades of History from the Deterioration of the Roman Empire (1439–1453). Tripartite periodization became standard after the German historian Christoph Cellarius published Universal History Divided into an Ancient, Medieval, and New Period (1683). -As economic and demographic methods were applied to the study of history, the trend was increasingly to see the late Middle Ages as a period of recession and crisis. Belgian historian Henri Pirenne continued the subdivision of Early, High, and Late Middle Ages in the years around World War I. Yet it was his Dutch colleague, Johan Huizinga, who was primarily responsible for popularising the pessimistic view of the Late Middle Ages, with his book The Autumn of the Middle Ages (1919). To Huizinga, whose research focused on France and the Low Countries rather than Italy, despair and decline were the main themes, not rebirth. -Modern historiography on the period has reached a consensus between the two extremes of innovation and crisis. It is now (generally) acknowledged that conditions were vastly different north and south of the Alps, and "Late Middle Ages" is often avoided entirely within Italian historiography. The term "Renaissance" is still considered useful for describing certain intellectual, cultural, or artistic developments, but not as the defining feature of an entire European historical epoch. The period from the early 14th century up until – and sometimes including – the 16th century, is rather seen as characterised by other trends: demographic and economic decline followed by recovery, the end of western religious unity and the subsequent emergence of the nation state, and the expansion of European influence onto the rest of the world. -After the failed union of Sweden and Norway of 1319–1365, the pan-Scandinavian Kalmar Union was instituted in 1397. The Swedes were reluctant members of the Danish-dominated union from the start. In an attempt to subdue the Swedes, King Christian II of Denmark had large numbers of the Swedish aristocracy killed in the Stockholm Bloodbath of 1520. Yet this measure only led to further hostilities, and Sweden broke away for good in 1523. Norway, on the other hand, became an inferior party of the union and remained united with Denmark until 1814. -Bohemia prospered in the 14th century, and the Golden Bull of 1356 made the king of Bohemia first among the imperial electors, but the Hussite revolution threw the country into crisis. The Holy Roman Empire passed to the Habsburgs in 1438, where it remained until its dissolution in 1806. Yet in spite of the extensive territories held by the Habsburgs, the Empire itself remained fragmented, and much real power and influence lay with the individual principalities. In addition, financial institutions, such as the Hanseatic League and the Fugger family, held great power, on both economic and a political levels. -Louis did not leave a son as heir after his death in 1382. Instead, he named as his heir the young prince Sigismund of Luxemburg, who was 11 years old. The Hungarian nobility did not accept his claim, and the result was an internal war. Sigismund eventually achieved total control of Hungary and established his court in Buda and Visegrád. Both palaces were rebuilt and improved, and were considered the richest of the time in Europe. Inheriting the throne of Bohemia and the Holy Roman Empire, Sigismund continued conducting his politics from Hungary, but he was kept busy fighting the Hussites and the Ottoman Empire, which was becoming a menace to Europe in the beginning of the 15th century. -The Bulgarian Empire was in decline by the 14th century, and the ascendancy of Serbia was marked by the Serbian victory over the Bulgarians in the Battle of Velbazhd in 1330. By 1346, the Serbian king Stefan Dušan had been proclaimed emperor. Yet Serbian dominance was short-lived; the Serbian army led by the Lazar Hrebljevanovic was defeated by the Ottomans at the Battle of Kosovo in 1389, where most of the Serbian nobility was killed and the south of the country came under Ottoman occupation, as much of southern Bulgaria had become Ottoman territory in 1371. Northern remnants of Bulgaria were finally conquered by 1396, Serbia fell in 1459, Bosnia in 1463, and Albania was finally subordinated in 1479 only a few years after the death of Skanderbeg. Belgrade, an Hungarian domain at the time, was the last large Balkan city to fall under Ottoman rule, in 1521. By the end of the medieval period, the entire Balkan peninsula was annexed by, or became vassal to, the Ottomans. -Avignon was the seat of the papacy from 1309 to 1376. With the return of the Pope to Rome in 1378, the Papal State developed into a major secular power, culminating in the morally corrupt papacy of Alexander VI. Florence grew to prominence amongst the Italian city-states through financial business, and the dominant Medici family became important promoters of the Renaissance through their patronage of the arts. Other city states in northern Italy also expanded their territories and consolidated their power, primarily Milan and Venice. The War of the Sicilian Vespers had by the early 14th century divided southern Italy into an Aragon Kingdom of Sicily and an Anjou Kingdom of Naples. In 1442, the two kingdoms were effectively united under Aragonese control. -The 1469 marriage of Isabella I of Castile and Ferdinand II of Aragon and the 1479 death of John II of Aragon led to the creation of modern-day Spain. In 1492, Granada was captured from the Moors, thereby completing the Reconquista. Portugal had during the 15th century – particularly under Henry the Navigator – gradually explored the coast of Africa, and in 1498, Vasco da Gama found the sea route to India. The Spanish monarchs met the Portuguese challenge by financing the expedition of Christopher Columbus to find a western sea route to India, leading to the discovery of the Americas in 1492. -Around 1300–1350 the Medieval Warm Period gave way to the Little Ice Age. The colder climate resulted in agricultural crises, the first of which is known as the Great Famine of 1315-1317. The demographic consequences of this famine, however, were not as severe as the plagues that occurred later in the century, particularly the Black Death. Estimates of the death rate caused by this epidemic range from one third to as much as sixty percent. By around 1420, the accumulated effect of recurring plagues and famines had reduced the population of Europe to perhaps no more than a third of what it was a century earlier. The effects of natural disasters were exacerbated by armed conflicts; this was particularly the case in France during the Hundred Years' War. -As the European population was severely reduced, land became more plentiful for the survivors, and labour consequently more expensive. Attempts by landowners to forcibly reduce wages, such as the English 1351 Statute of Laborers, were doomed to fail. These efforts resulted in nothing more than fostering resentment among the peasantry, leading to rebellions such as the French Jacquerie in 1358 and the English Peasants' Revolt in 1381. The long-term effect was the virtual end of serfdom in Western Europe. In Eastern Europe, on the other hand, landowners were able to exploit the situation to force the peasantry into even more repressive bondage. -Up until the mid-14th century, Europe had experienced steadily increasing urbanisation. Cities were also decimated by the Black Death, but the role of urban areas as centres of learning, commerce and government ensured continued growth. By 1500, Venice, Milan, Naples, Paris and Constantinople each probably had more than 100,000 inhabitants. Twenty-two other cities were larger than 40,000; most of these were in Italy and the Iberian peninsula, but there were also some in France, the Empire, the Low Countries, plus London in England. -Changes also took place within the recruitment and composition of armies. The use of the national or feudal levy was gradually replaced by paid troops of domestic retinues or foreign mercenaries. The practice was associated with Edward III of England and the condottieri of the Italian city-states. All over Europe, Swiss soldiers were in particularly high demand. At the same time, the period also saw the emergence of the first permanent armies. It was in Valois France, under the heavy demands of the Hundred Years' War, that the armed forces gradually assumed a permanent nature. -Parallel to the military developments emerged also a constantly more elaborate chivalric code of conduct for the warrior class. This new-found ethos can be seen as a response to the diminishing military role of the aristocracy, and gradually it became almost entirely detached from its military origin. The spirit of chivalry was given expression through the new (secular) type of chivalric orders; the first of these was the Order of St. George, founded by Charles I of Hungary in 1325, while the best known was probably the English Order of the Garter, founded by Edward III in 1348. -The French crown's increasing dominance over the Papacy culminated in the transference of the Holy See to Avignon in 1309. When the Pope returned to Rome in 1377, this led to the election of different popes in Avignon and Rome, resulting in the Papal Schism (1378–1417). The Schism divided Europe along political lines; while France, her ally Scotland and the Spanish kingdoms supported the Avignon Papacy, France's enemy England stood behind the Pope in Rome, together with Portugal, Scandinavia and most of the German princes. -Though many of the events were outside the traditional time-period of the Middle Ages, the end of the unity of the Western Church (the Protestant Reformation), was one of the distinguishing characteristics of the medieval period. The Catholic Church had long fought against heretic movements, but during the Late Middle Ages, it started to experience demands for reform from within. The first of these came from Oxford professor John Wycliffe in England. Wycliffe held that the Bible should be the only authority in religious questions, and he spoke out against transubstantiation, celibacy and indulgences. In spite of influential supporters among the English aristocracy, such as John of Gaunt, the movement was not allowed to survive. Though Wycliffe himself was left unmolested, his supporters, the Lollards, were eventually suppressed in England. -The marriage of Richard II of England to Anne of Bohemia established contacts between the two nations and brought Lollard ideas to her homeland. The teachings of the Czech priest Jan Hus were based on those of John Wycliffe, yet his followers, the Hussites, were to have a much greater political impact than the Lollards. Hus gained a great following in Bohemia, and in 1414, he was requested to appear at the Council of Constance to defend his cause. When he was burned as a heretic in 1415, it caused a popular uprising in the Czech lands. The subsequent Hussite Wars fell apart due to internal quarrels and did not result in religious or national independence for the Czechs, but both the Catholic Church and the German element within the country were weakened. -Martin Luther, a German monk, started the German Reformation by posting 95 theses on the castle church of Wittenberg on October 31, 1517. The immediate provocation spurring this act was Pope Leo X’s renewal of the indulgence for the building of the new St. Peter's Basilica in 1514. Luther was challenged to recant his heresy at the Diet of Worms in 1521. When he refused, he was placed under the ban of the Empire by Charles V. Receiving the protection of Frederick the Wise, he was then able to translate the Bible into German. -In the late 13th and early 14th centuries, a process took place – primarily in Italy but partly also in the Empire – that historians have termed a 'commercial revolution'. Among the innovations of the period were new forms of partnership and the issuing of insurance, both of which contributed to reducing the risk of commercial ventures; the bill of exchange and other forms of credit that circumvented the canonical laws for gentiles against usury, and eliminated the dangers of carrying bullion; and new forms of accounting, in particular double-entry bookkeeping, which allowed for better oversight and accuracy. -With the financial expansion, trading rights became more jealously guarded by the commercial elite. Towns saw the growing power of guilds, while on a national level special companies would be granted monopolies on particular trades, like the English wool Staple. The beneficiaries of these developments would accumulate immense wealth. Families like the Fuggers in Germany, the Medicis in Italy, the de la Poles in England, and individuals like Jacques Coeur in France would help finance the wars of kings, and achieve great political influence in the process. -Though there is no doubt that the demographic crisis of the 14th century caused a dramatic fall in production and commerce in absolute terms, there has been a vigorous historical debate over whether the decline was greater than the fall in population. While the older orthodoxy held that the artistic output of the Renaissance was a result of greater opulence, more recent studies have suggested that there might have been a so-called 'depression of the Renaissance'. In spite of convincing arguments for the case, the statistical evidence is simply too incomplete for a definite conclusion to be made. -The predominant school of thought in the 13th century was the Thomistic reconciliation of the teachings of Aristotle with Christian theology. The Condemnation of 1277, enacted at the University of Paris, placed restrictions on ideas that could be interpreted as heretical; restrictions that had implication for Aristotelian thought. An alternative was presented by William of Ockham, who insisted that the world of reason and the world of faith had to be kept apart. Ockham introduced the principle of parsimony – or Occam's razor – whereby a simple theory is preferred to a more complex one, and speculation on unobservable phenomena is avoided. -This new approach liberated scientific speculation from the dogmatic restraints of Aristotelian science, and paved the way for new approaches. Particularly within the field of theories of motion great advances were made, when such scholars as Jean Buridan, Nicole Oresme and the Oxford Calculators challenged the work of Aristotle. Buridan developed the theory of impetus as the cause of the motion of projectiles, which was an important step towards the modern concept of inertia. The works of these scholars anticipated the heliocentric worldview of Nicolaus Copernicus. -Certain technological inventions of the period – whether of Arab or Chinese origin, or unique European innovations – were to have great influence on political and social developments, in particular gunpowder, the printing press and the compass. The introduction of gunpowder to the field of battle affected not only military organisation, but helped advance the nation state. Gutenberg's movable type printing press made possible not only the Reformation, but also a dissemination of knowledge that would lead to a gradually more egalitarian society. The compass, along with other innovations such as the cross-staff, the mariner's astrolabe, and advances in shipbuilding, enabled the navigation of the World Oceans, and the early phases of colonialism. Other inventions had a greater impact on everyday life, such as eyeglasses and the weight-driven clock. -The period saw several important technical innovations, like the principle of linear perspective found in the work of Masaccio, and later described by Brunelleschi. Greater realism was also achieved through the scientific study of anatomy, championed by artists like Donatello. This can be seen particularly well in his sculptures, inspired by the study of classical models. As the centre of the movement shifted to Rome, the period culminated in the High Renaissance masters da Vinci, Michelangelo and Raphael. -The ideas of the Italian Renaissance were slow to cross the Alps into northern Europe, but important artistic innovations were made also in the Low Countries. Though not – as previously believed – the inventor of oil painting, Jan van Eyck was a champion of the new medium, and used it to create works of great realism and minute detail. The two cultures influenced each other and learned from each other, but painting in the Netherlands remained more focused on textures and surfaces than the idealized compositions of Italy. -Dante Alighieri's Divine Comedy, written in the early 14th century, merged a medieval world view with classical ideals. Another promoter of the Italian language was Boccaccio with his Decameron. The application of the vernacular did not entail a rejection of Latin, and both Dante and Boccaccio wrote prolifically in Latin as well as Italian, as would Petrarch later (whose Canzoniere also promoted the vernacular and whose contents are considered the first modern lyric poems). Together the three poets established the Tuscan dialect as the norm for the modern Italian language. -Music was an important part of both secular and spiritual culture, and in the universities it made up part of the quadrivium of the liberal arts. From the early 13th century, the dominant sacred musical form had been the motet; a composition with text in several parts. From the 1330s and onwards, emerged the polyphonic style, which was a more complex fusion of independent voices. Polyphony had been common in the secular music of the Provençal troubadours. Many of these had fallen victim to the 13th-century Albigensian Crusade, but their influence reached the papal court at Avignon. -The main representatives of the new style, often referred to as ars nova as opposed to the ars antiqua, were the composers Philippe de Vitry and Guillaume de Machaut. In Italy, where the Provençal troubadours had also found refuge, the corresponding period goes under the name of trecento, and the leading composers were Giovanni da Cascia, Jacopo da Bologna and Francesco Landini. Prominent reformer of Orthodox Church music from the first half of 14th century was John Kukuzelis; he also introduced a system of notation widely used in the Balkans in the following centuries. -Morality plays emerged as a distinct dramatic form around 1400 and flourished until 1550. The most interesting morality play is The Castle of Perseverance which depicts mankind's progress from birth to death. However, the most famous morality play and perhaps best known medieval drama is Everyman. Everyman receives Death's summons, struggles to escape and finally resigns himself to necessity. Along the way, he is deserted by Kindred, Goods, and Fellowship - only Good Deeds goes with him to the grave. -At the end of the Late Middle Ages, professional actors began to appear in England and Europe. Richard III and Henry VII both maintained small companies of professional actors. Their plays were performed in the Great Hall of a nobleman's residence, often with a raised platform at one end for the audience and a "screen" at the other for the actors. Also important were Mummers' plays, performed during the Christmas season, and court masques. These masques were especially popular during the reign of Henry VIII who had a House of Revels built and an Office of Revels established in 1545. -The end of medieval drama came about due to a number of factors, including the weakening power of the Catholic Church, the Protestant Reformation and the banning of religious plays in many countries. Elizabeth I forbid all religious plays in 1558 and the great cycle plays had been silenced by the 1580s. Similarly, religious plays were banned in the Netherlands in 1539, the Papal States in 1547 and in Paris in 1548. The abandonment of these plays destroyed the international theatre that had thereto existed and forced each country to develop its own form of drama. It also allowed dramatists to turn to secular subjects and the reviving interest in Greek and Roman theatre provided them with the perfect opportunity. -After the end of the late Middle Ages period, the Renaissance would spread unevenly over continental Europe from the southern European region. The intellectual transformation of the Renaissance is viewed as a bridge between the Middle Ages and the Modern era. Europeans would later begin an era of world discovery. Combined with the influx of classical ideas was the invention of printing which facilitated dissemination of the printed word and democratized learning. These two things would lead to the Protestant Reformation. Europeans also discovered new trading routes, as was the case with Columbus’s travel to the Americas in 1492, and Vasco da Gama’s circumnavigation of Africa and India in 1498. Their discoveries strengthened the economy and power of European nations. -At the end of the 15th century the Ottoman Empire advanced all over Southeastern Europe, eventually conquering the Byzantine Empire and extending control over the Balkan states. Hungary was the last bastion of the Latin Christian world in the East, and fought to keep its rule over a period of two centuries. After the tragic death of the young king Vladislaus I of Hungary during the Battle of Varna in 1444 against the Ottomans, the Kingdom was placed in the hands of count John Hunyadi, who became Hungary's regent-governor (1446–1453). Hunyadi was considered one of the most relevant military figures of the 15th century: Pope Pius II awarded him the title of Athleta Christi or Champion of Christ for being the only hope of resisting the Ottomans from advancing to Central and Western Europe. -Hunyadi succeeded during the Siege of Belgrade in 1456 against the Ottomans, the biggest victory against that empire in decades. This battle became a real Crusade against the Muslims, as the peasants were motivated by the Franciscan monk Saint John of Capistrano, who came from Italy predicating Holy War. The effect that it created in that time was one of the main factors that helped in achieving the victory. However the premature death of the Hungarian Lord left Pannonia defenseless and in chaos. In an extremely unusual event for the Middle Ages, Hunyadi's son, Matthias, was elected as King of Hungary by the nobility. For the first time, a member of an aristocratic family (and not from a royal family) was crowned. -King Matthias Corvinus of Hungary (1458–1490) was one of the most prominent figures of the period, directing campaigns to the West, conquering Bohemia in answer to the Pope's call for help against the Hussite Protestants. Also, in resolving political hostilities with the German emperor Frederick III of Habsburg, he invaded his western domains. Matthew organized the Black Army of mercenary soldiers; it was considered as the biggest army of its time. Using this powerful tool, the Hungarian king led wars against the Turkish armies and stopped the Ottomans during his reign. After the death of Matthew, and with end of the Black Army, the Ottoman Empire grew in strength and Central Europe was defenseless. At the Battle of Mohács, the forces of the Ottoman Empire annihilated the Hungarian army and Louis II of Hungary drowned in the Csele Creek while trying to escape. The leader of the Hungarian army, Pál Tomori, also died in the battle. This is considered to be one of the final battles of Medieval times. -The changes brought about by these developments have led many scholars to view this period as the end of the Middle Ages and beginning of modern history and early modern Europe. However, the division is somewhat artificial, since ancient learning was never entirely absent from European society. As a result there was developmental continuity between the ancient age (via classical antiquity) and the modern age. Some historians, particularly in Italy, prefer not to speak of the late Middle Ages at all, but rather see the high period of the Middle Ages transitioning to the Renaissance and the modern era. diff --git a/e2e/samples/test_documents/Normans.txt b/e2e/samples/test_documents/Normans.txt deleted file mode 100644 index d8f2f31e7b..0000000000 --- a/e2e/samples/test_documents/Normans.txt +++ /dev/null @@ -1,45 +0,0 @@ -The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse ("Norman" comes from "Norseman") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries. -The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands. -The English name "Normans" comes from the French words Normans/Normanz, plural of Normant, modern French normand, which is itself borrowed from Old Low Franconian Nortmann "Northman" or directly from Old Norse Norðmaðr, Latinized variously as Nortmannus, Normannus, or Nordmannus (recorded in Medieval Latin, 9th century) to mean "Norseman, Viking". -In the course of the 10th century, the initially destructive incursions of Norse war bands into the rivers of France evolved into more permanent encampments that included local women and personal property. The Duchy of Normandy, which began in 911 as a fiefdom, was established by the treaty of Saint-Clair-sur-Epte between King Charles III of West Francia and the famed Viking ruler Rollo, and was situated in the former Frankish kingdom of Neustria. The treaty offered Rollo and his men the French lands between the river Epte and the Atlantic coast in exchange for their protection against further Viking incursions. The area corresponded to the northern part of present-day Upper Normandy down to the river Seine, but the Duchy would eventually extend west beyond the Seine. The territory was roughly equivalent to the old province of Rouen, and reproduced the Roman administrative structure of Gallia Lugdunensis II (part of the former Gallia Lugdunensis). -Before Rollo's arrival, its populations did not differ from Picardy or the Île-de-France, which were considered "Frankish". Earlier Viking settlers had begun arriving in the 880s, but were divided between colonies in the east (Roumois and Pays de Caux) around the low Seine valley and in the west in the Cotentin Peninsula, and were separated by traditional pagii, where the population remained about the same with almost no foreign settlers. Rollo's contingents who raided and ultimately settled Normandy and parts of the Atlantic coast included Danes, Norwegians, Norse–Gaels, Orkney Vikings, possibly Swedes, and Anglo-Danes from the English Danelaw under Norse control. -The descendants of Rollo's Vikings and their Frankish wives would replace the Norse religion and Old Norse language with Catholicism (Christianity) and the Gallo-Romance language of the local people, blending their maternal Frankish heritage with Old Norse traditions and customs to synthesize a unique "Norman" culture in the north of France. The Norman language was forged by the adoption of the indigenous langue d'oïl branch of Romance by a Norse-speaking ruling class, and it developed into the regional language that survives today. -The Normans thereafter adopted the growing feudal doctrines of the rest of France, and worked them into a functional hierarchical system in both Normandy and in England. The new Norman rulers were culturally and ethnically distinct from the old French aristocracy, most of whom traced their lineage to Franks of the Carolingian dynasty. Most Norman knights remained poor and land-hungry, and by 1066 Normandy had been exporting fighting horsemen for more than a generation. Many Normans of Italy, France and England eventually served as avid Crusaders under the Italo-Norman prince Bohemund I and the Anglo-Norman king Richard the Lion-Heart. -Opportunistic bands of Normans successfully established a foothold in Southern Italy (the Mezzogiorno). Probably as the result of returning pilgrims' stories, the Normans entered the Mezzogiorno as warriors in 1017 at the latest. In 999, according to Amatus of Montecassino, Norman pilgrims returning from Jerusalem called in at the port of Salerno when a Saracen attack occurred. The Normans fought so valiantly that Prince Guaimar III begged them to stay, but they refused and instead offered to tell others back home of the prince's request. William of Apulia tells that, in 1016, Norman pilgrims to the shrine of the Archangel Michael at Monte Gargano were met by Melus of Bari, a Lombard nobleman and rebel, who persuaded them to return with more warriors to help throw off the Byzantine rule, which they did. -The two most prominent Norman families to arrive in the Mediterranean were descendants of Tancred of Hauteville and the Drengot family, of whom Rainulf Drengot received the county of Aversa, the first Norman toehold in the south, from Duke Sergius IV of Naples in 1030. The Hauteville family achieved princely rank by proclaiming prince Guaimar IV of Salerno "Duke of Apulia and Calabria". He promptly awarded their elected leader, William Iron Arm, with the title of count in his capital of Melfi. The Drengot family thereafter attained the principality of Capua, and emperor Henry III legally ennobled the Hauteville leader, Drogo, as "dux et magister Italiae comesque Normannorum totius Apuliae et Calabriae" ("Duke and Master of Italy and Count of the Normans of all Apulia and Calabria") in 1047. -From these bases, the Normans eventually captured Sicily and Malta from the Saracens, under the leadership of the famous Robert Guiscard, a Hauteville, and his younger brother Roger the Great Count. Roger's son, Roger II of Sicily, was crowned king in 1130 (exactly one century after Rainulf was "crowned" count) by Antipope Anacletus II. The Kingdom of Sicily lasted until 1194, when it was transferred to the House of Hohenstaufen through marriage. The Normans left their legacy in many castles, such as William Iron Arm's citadel at Squillace, and cathedrals, such as Roger II's Cappella Palatina chapel at Palermo, which dot the landscape and give a wholly distinct architectural flavor to accompany its unique history. -Institutionally, the Normans combined the administrative machinery of the Byzantines, Arabs, and Lombards with their own conceptions of feudal law and order to forge a unique government. Under this state, there was great religious freedom, and alongside the Norman nobles existed a meritocratic bureaucracy of Jews, Muslims and Christians, both Catholic and Eastern Orthodox. The Kingdom of Sicily thus became characterized by Norman, Byzantine Greek, Arab, Lombard and "native" Sicilian populations living in harmony, and its Norman rulers fostered plans of establishing an Empire that would have encompassed Fatimid Egypt as well as the Crusader states in the Levant. One of the great geographical treatises of the Middle Ages, the "Tabula Rogeriana", was written by the Andalusian al-Idrisi for king Roger II of Sicily, and entitled "Kitab Rudjdjar" ("The Book of Roger"). -Soon after the Normans began to enter Italy, they entered the Byzantine Empire and then Armenia, fighting against the Pechenegs, the Bulgars, and especially the Seljuk Turks. Norman mercenaries were first encouraged to come to the south by the Lombards to act against the Byzantines, but they soon fought in Byzantine service in Sicily. They were prominent alongside Varangian and Lombard contingents in the Sicilian campaign of George Maniaces in 1038–40. There is debate whether the Normans in Greek service actually were from Norman Italy, and it now seems likely only a few came from there. It is also unknown how many of the "Franks", as the Byzantines called them, were Normans and not other Frenchmen. -One of the first Norman mercenaries to serve as a Byzantine general was Hervé in the 1050s. By then however, there were already Norman mercenaries serving as far away as Trebizond and Georgia. They were based at Malatya and Edessa, under the Byzantine duke of Antioch, Isaac Komnenos. In the 1060s, Robert Crispin led the Normans of Edessa against the Turks. Roussel de Bailleul even tried to carve out an independent state in Asia Minor with support from the local population, but he was stopped by the Byzantine general Alexius Komnenos. -Some Normans joined Turkish forces to aid in the destruction of the Armenians vassal-states of Sassoun and Taron in far eastern Anatolia. Later, many took up service with the Armenian state further south in Cilicia and the Taurus Mountains. A Norman named Oursel led a force of "Franks" into the upper Euphrates valley in northern Syria. From 1073 to 1074, 8,000 of the 20,000 troops of the Armenian general Philaretus Brachamius were Normans—formerly of Oursel—led by Raimbaud. They even lent their ethnicity to the name of their castle: Afranji, meaning "Franks." The known trade between Amalfi and Antioch and between Bari and Tarsus may be related to the presence of Italo-Normans in those cities while Amalfi and Bari were under Norman rule in Italy. -Several families of Byzantine Greece were of Norman mercenary origin during the period of the Comnenian Restoration, when Byzantine emperors were seeking out western European warriors. The Raoulii were descended from an Italo-Norman named Raoul, the Petraliphae were descended from a Pierre d'Aulps, and that group of Albanian clans known as the Maniakates were descended from Normans who served under George Maniaces in the Sicilian expedition of 1038. -Robert Guiscard, an other Norman adventurer previously elevated to the dignity of count of Apulia as the result of his military successes, ultimately drove the Byzantines out of southern Italy. Having obtained the consent of pope Gregory VII and acting as his vassal, Robert continued his campaign conquering the Balkan peninsula as a foothold for western feudal lords and the Catholic Church. After allying himself with Croatia and the Catholic cities of Dalmatia, in 1081 he led an army of 30,000 men in 300 ships landing on the southern shores of Albania, capturing Valona, Kanina, Jericho (Orikumi), and reaching Butrint after numerous pillages. They joined the fleet that had previously conquered Corfu and attacked Dyrrachium from land and sea, devastating everything along the way. Under these harsh circumstances, the locals accepted the call of emperor Alexius I Comnenus to join forces with the Byzantines against the Normans. The Albanian forces could not take part in the ensuing battle because it had started before their arrival. Immediately before the battle, the Venetian fleet had secured a victory in the coast surrounding the city. Forced to retreat, Alexius ceded the command to a high Albanian official named Comiscortes in the service of Byzantium. The city's garrison resisted until February 1082, when Dyrrachium was betrayed to the Normans by the Venetian and Amalfitan merchants who had settled there. The Normans were now free to penetrate into the hinterland; they took Ioannina and some minor cities in southwestern Macedonia and Thessaly before appearing at the gates of Thessalonica. Dissension among the high ranks coerced the Normans to retreat to Italy. They lost Dyrrachium, Valona, and Butrint in 1085, after the death of Robert. -A few years after the First Crusade, in 1107, the Normans under the command of Bohemond, Robert's son, landed in Valona and besieged Dyrrachium using the most sophisticated military equipment of the time, but to no avail. Meanwhile, they occupied Petrela, the citadel of Mili at the banks of the river Deabolis, Gllavenica (Ballsh), Kanina and Jericho. This time, the Albanians sided with the Normans, dissatisfied by the heavy taxes the Byzantines had imposed upon them. With their help, the Normans secured the Arbanon passes and opened their way to Dibra. The lack of supplies, disease and Byzantine resistance forced Bohemond to retreat from his campaign and sign a peace treaty with the Byzantines in the city of Deabolis. -The further decline of Byzantine state-of-affairs paved the road to a third attack in 1185, when a large Norman army invaded Dyrrachium, owing to the betrayal of high Byzantine officials. Some time later, Dyrrachium—one of the most important naval bases of the Adriatic—fell again to Byzantine hands. -The Normans were in contact with England from an early date. Not only were their original Viking brethren still ravaging the English coasts, they occupied most of the important ports opposite England across the English Channel. This relationship eventually produced closer ties of blood through the marriage of Emma, sister of Duke Richard II of Normandy, and King Ethelred II of England. Because of this, Ethelred fled to Normandy in 1013, when he was forced from his kingdom by Sweyn Forkbeard. His stay in Normandy (until 1016) influenced him and his sons by Emma, who stayed in Normandy after Cnut the Great's conquest of the isle. -When finally Edward the Confessor returned from his father's refuge in 1041, at the invitation of his half-brother Harthacnut, he brought with him a Norman-educated mind. He also brought many Norman counsellors and fighters, some of whom established an English cavalry force. This concept never really took root, but it is a typical example of the attitudes of Edward. He appointed Robert of Jumièges archbishop of Canterbury and made Ralph the Timid earl of Hereford. He invited his brother-in-law Eustace II, Count of Boulogne to his court in 1051, an event which resulted in the greatest of early conflicts between Saxon and Norman and ultimately resulted in the exile of Earl Godwin of Wessex. -In 1066, Duke William II of Normandy conquered England killing King Harold II at the Battle of Hastings. The invading Normans and their descendants replaced the Anglo-Saxons as the ruling class of England. The nobility of England were part of a single Normans culture and many had lands on both sides of the channel. Early Norman kings of England, as Dukes of Normandy, owed homage to the King of France for their land on the continent. They considered England to be their most important holding (it brought with it the title of King—an important status symbol). -Eventually, the Normans merged with the natives, combining languages and traditions. In the course of the Hundred Years' War, the Norman aristocracy often identified themselves as English. The Anglo-Norman language became distinct from the Latin language, something that was the subject of some humour by Geoffrey Chaucer. The Anglo-Norman language was eventually absorbed into the Anglo-Saxon language of their subjects (see Old English) and influenced it, helping (along with the Norse language of the earlier Anglo-Norse settlers and the Latin used by the church) in the development of Middle English. It in turn evolved into Modern English. -The Normans had a profound effect on Irish culture and history after their invasion at Bannow Bay in 1169. Initially the Normans maintained a distinct culture and ethnicity. Yet, with time, they came to be subsumed into Irish culture to the point that it has been said that they became "more Irish than the Irish themselves." The Normans settled mostly in an area in the east of Ireland, later known as the Pale, and also built many fine castles and settlements, including Trim Castle and Dublin Castle. Both cultures intermixed, borrowing from each other's language, culture and outlook. Norman descendants today can be recognised by their surnames. Names such as French, (De) Roche, Devereux, D'Arcy, Treacy and Lacy are particularly common in the southeast of Ireland, especially in the southern part of County Wexford where the first Norman settlements were established. Other Norman names such as Furlong predominate there. Another common Norman-Irish name was Morell (Murrell) derived from the French Norman name Morel. Other names beginning with Fitz (from the Norman for son) indicate Norman ancestry. These included Fitzgerald, FitzGibbons (Gibbons) dynasty, Fitzmaurice. Other families bearing such surnames as Barry (de Barra) and De Búrca (Burke) are also of Norman extraction. -One of the claimants of the English throne opposing William the Conqueror, Edgar Atheling, eventually fled to Scotland. King Malcolm III of Scotland married Edgar's sister Margaret, and came into opposition to William who had already disputed Scotland's southern borders. William invaded Scotland in 1072, riding as far as Abernethy where he met up with his fleet of ships. Malcolm submitted, paid homage to William and surrendered his son Duncan as a hostage, beginning a series of arguments as to whether the Scottish Crown owed allegiance to the King of England. -Normans came into Scotland, building castles and founding noble families who would provide some future kings, such as Robert the Bruce, as well as founding a considerable number of the Scottish clans. King David I of Scotland, whose elder brother Alexander I had married Sybilla of Normandy, was instrumental in introducing Normans and Norman culture to Scotland, part of the process some scholars call the "Davidian Revolution". Having spent time at the court of Henry I of England (married to David's sister Maud of Scotland), and needing them to wrestle the kingdom from his half-brother Máel Coluim mac Alaxandair, David had to reward many with lands. The process was continued under David's successors, most intensely of all under William the Lion. The Norman-derived feudal system was applied in varying degrees to most of Scotland. Scottish families of the names Bruce, Gray, Ramsay, Fraser, Ogilvie, Montgomery, Sinclair, Pollock, Burnard, Douglas and Gordon to name but a few, and including the later royal House of Stewart, can all be traced back to Norman ancestry. -Even before the Norman Conquest of England, the Normans had come into contact with Wales. Edward the Confessor had set up the aforementioned Ralph as earl of Hereford and charged him with defending the Marches and warring with the Welsh. In these original ventures, the Normans failed to make any headway into Wales. -Subsequent to the Conquest, however, the Marches came completely under the dominance of William's most trusted Norman barons, including Bernard de Neufmarché, Roger of Montgomery in Shropshire and Hugh Lupus in Cheshire. These Normans began a long period of slow conquest during which almost all of Wales was at some point subject to Norman interference. Norman words, such as baron (barwn), first entered Welsh at that time. -The legendary religious zeal of the Normans was exercised in religious wars long before the First Crusade carved out a Norman principality in Antioch. They were major foreign participants in the Reconquista in Iberia. In 1018, Roger de Tosny travelled to the Iberian Peninsula to carve out a state for himself from Moorish lands, but failed. In 1064, during the War of Barbastro, William of Montreuil led the papal army and took a huge booty. -In 1096, Crusaders passing by the siege of Amalfi were joined by Bohemond of Taranto and his nephew Tancred with an army of Italo-Normans. Bohemond was the de facto leader of the Crusade during its passage through Asia Minor. After the successful Siege of Antioch in 1097, Bohemond began carving out an independent principality around that city. Tancred was instrumental in the conquest of Jerusalem and he worked for the expansion of the Crusader kingdom in Transjordan and the region of Galilee.[citation needed] -The conquest of Cyprus by the Anglo-Norman forces of the Third Crusade opened a new chapter in the history of the island, which would be under Western European domination for the following 380 years. Although not part of a planned operation, the conquest had much more permanent results than initially expected. -In April 1191 Richard the Lion-hearted left Messina with a large fleet in order to reach Acre. But a storm dispersed the fleet. After some searching, it was discovered that the boat carrying his sister and his fiancée Berengaria was anchored on the south coast of Cyprus, together with the wrecks of several other ships, including the treasure ship. Survivors of the wrecks had been taken prisoner by the island's despot Isaac Komnenos. On 1 May 1191, Richard's fleet arrived in the port of Limassol on Cyprus. He ordered Isaac to release the prisoners and the treasure. Isaac refused, so Richard landed his troops and took Limassol. -Various princes of the Holy Land arrived in Limassol at the same time, in particular Guy de Lusignan. All declared their support for Richard provided that he support Guy against his rival Conrad of Montferrat. The local barons abandoned Isaac, who considered making peace with Richard, joining him on the crusade, and offering his daughter in marriage to the person named by Richard. But Isaac changed his mind and tried to escape. Richard then proceeded to conquer the whole island, his troops being led by Guy de Lusignan. Isaac surrendered and was confined with silver chains, because Richard had promised that he would not place him in irons. By 1 June, Richard had conquered the whole island. His exploit was well publicized and contributed to his reputation; he also derived significant financial gains from the conquest of the island. Richard left for Acre on 5 June, with his allies. Before his departure, he named two of his Norman generals, Richard de Camville and Robert de Thornham, as governors of Cyprus. -While in Limassol, Richard the Lion-Heart married Berengaria of Navarre, first-born daughter of King Sancho VI of Navarre. The wedding was held on 12 May 1191 at the Chapel of St. George and it was attended by Richard's sister Joan, whom he had brought from Sicily. The marriage was celebrated with great pomp and splendor. Among other grand ceremonies was a double coronation: Richard caused himself to be crowned King of Cyprus, and Berengaria Queen of England and Queen of Cyprus as well. -The rapid Anglo-Norman conquest proved more important than it seemed. The island occupied a key strategic position on the maritime lanes to the Holy Land, whose occupation by the Christians could not continue without support from the sea. Shortly after the conquest, Cyprus was sold to the Knights Templar and it was subsequently acquired, in 1192, by Guy de Lusignan and became a stable feudal kingdom. It was only in 1489 that the Venetians acquired full control of the island, which remained a Christian stronghold until the fall of Famagusta in 1571. -Between 1402 and 1405, the expedition led by the Norman noble Jean de Bethencourt and the Poitevine Gadifer de la Salle conquered the Canarian islands of Lanzarote, Fuerteventura and El Hierro off the Atlantic coast of Africa. Their troops were gathered in Normandy, Gascony and were later reinforced by Castilian colonists. -Bethencourt took the title of King of the Canary Islands, as vassal to Henry III of Castile. In 1418, Jean's nephew Maciot de Bethencourt sold the rights to the islands to Enrique Pérez de Guzmán, 2nd Count de Niebla. -The customary law of Normandy was developed between the 10th and 13th centuries and survives today through the legal systems of Jersey and Guernsey in the Channel Islands. Norman customary law was transcribed in two customaries in Latin by two judges for use by them and their colleagues: These are the Très ancien coutumier (Very ancient customary), authored between 1200 and 1245; and the Grand coutumier de Normandie (Great customary of Normandy, originally Summa de legibus Normanniae in curia laïcali), authored between 1235 and 1245. -Norman architecture typically stands out as a new stage in the architectural history of the regions they subdued. They spread a unique Romanesque idiom to England and Italy, and the encastellation of these regions with keeps in their north French style fundamentally altered the military landscape. Their style was characterised by rounded arches, particularly over windows and doorways, and massive proportions. -In England, the period of Norman architecture immediately succeeds that of the Anglo-Saxon and precedes the Early Gothic. In southern Italy, the Normans incorporated elements of Islamic, Lombard, and Byzantine building techniques into their own, initiating a unique style known as Norman-Arab architecture within the Kingdom of Sicily. -In the visual arts, the Normans did not have the rich and distinctive traditions of the cultures they conquered. However, in the early 11th century the dukes began a programme of church reform, encouraging the Cluniac reform of monasteries and patronising intellectual pursuits, especially the proliferation of scriptoria and the reconstitution of a compilation of lost illuminated manuscripts. The church was utilised by the dukes as a unifying force for their disparate duchy. The chief monasteries taking part in this "renaissance" of Norman art and scholarship were Mont-Saint-Michel, Fécamp, Jumièges, Bec, Saint-Ouen, Saint-Evroul, and Saint-Wandrille. These centres were in contact with the so-called "Winchester school", which channeled a pure Carolingian artistic tradition to Normandy. In the final decade of the 11th and first of the 12th century, Normandy experienced a golden age of illustrated manuscripts, but it was brief and the major scriptoria of Normandy ceased to function after the midpoint of the century. -The French Wars of Religion in the 16th century and French Revolution in the 18th successively destroyed much of what existed in the way of the architectural and artistic remnant of this Norman creativity. The former, with their violence, caused the wanton destruction of many Norman edifices; the latter, with its assault on religion, caused the purposeful destruction of religious objects of any type, and its destabilisation of society resulted in rampant pillaging. -By far the most famous work of Norman art is the Bayeux Tapestry, which is not a tapestry but a work of embroidery. It was commissioned by Odo, the Bishop of Bayeux and first Earl of Kent, employing natives from Kent who were learned in the Nordic traditions imported in the previous half century by the Danish Vikings. -In Britain, Norman art primarily survives as stonework or metalwork, such as capitals and baptismal fonts. In southern Italy, however, Norman artwork survives plentifully in forms strongly influenced by its Greek, Lombard, and Arab forebears. Of the royal regalia preserved in Palermo, the crown is Byzantine in style and the coronation cloak is of Arab craftsmanship with Arabic inscriptions. Many churches preserve sculptured fonts, capitals, and more importantly mosaics, which were common in Norman Italy and drew heavily on the Greek heritage. Lombard Salerno was a centre of ivorywork in the 11th century and this continued under Norman domination. Finally should be noted the intercourse between French Crusaders traveling to the Holy Land who brought with them French artefacts with which to gift the churches at which they stopped in southern Italy amongst their Norman cousins. For this reason many south Italian churches preserve works from France alongside their native pieces. -Normandy was the site of several important developments in the history of classical music in the 11th century. Fécamp Abbey and Saint-Evroul Abbey were centres of musical production and education. At Fécamp, under two Italian abbots, William of Volpiano and John of Ravenna, the system of denoting notes by letters was developed and taught. It is still the most common form of pitch representation in English- and German-speaking countries today. Also at Fécamp, the staff, around which neumes were oriented, was first developed and taught in the 11th century. Under the German abbot Isembard, La Trinité-du-Mont became a centre of musical composition. -At Saint Evroul, a tradition of singing had developed and the choir achieved fame in Normandy. Under the Norman abbot Robert de Grantmesnil, several monks of Saint-Evroul fled to southern Italy, where they were patronised by Robert Guiscard and established a Latin monastery at Sant'Eufemia. There they continued the tradition of singing. diff --git a/haystack/components/generators/chat/openai.py b/haystack/components/generators/chat/openai.py index 8369892191..5a71ac595f 100644 --- a/haystack/components/generators/chat/openai.py +++ b/haystack/components/generators/chat/openai.py @@ -250,7 +250,7 @@ def _connect_chunks(self, chunk: Any, chunks: List[StreamingChunk]) -> ChatMessa complete_response = ChatMessage.from_assistant("".join([chunk.content for chunk in chunks])) complete_response.meta.update( { - "model": chunk.model, + "model": chunk.embeddings_model, "index": 0, "finish_reason": chunk.choices[0].finish_reason, "usage": {}, # we don't have usage data for streaming responses diff --git a/haystack/evaluation/eval_run_result.py b/haystack/evaluation/eval_run_result.py index c0bc2daa93..f48c01138c 100644 --- a/haystack/evaluation/eval_run_result.py +++ b/haystack/evaluation/eval_run_result.py @@ -119,7 +119,7 @@ def comparative_individual_scores_report(self, other: "BaseEvaluationRunResult") this_name = f"{this_name}_first" other_name = f"{other_name}_second" - if self.inputs != other.inputs: + if self.inputs.keys() != other.inputs.keys(): warn(f"The inputs to the two evaluation results differ; using the inputs of '{this_name}'.") pipe_a_df = self.to_pandas() diff --git a/proposals/text/4284-drop-basecomponent.md b/proposals/text/4284-drop-basecomponent.md index 8ed961b6cb..7fc5c18095 100644 --- a/proposals/text/4284-drop-basecomponent.md +++ b/proposals/text/4284-drop-basecomponent.md @@ -533,103 +533,108 @@ A Haystack node is any class that abides the following contract: @node class MyNode: - def __init__(self, model_name: str: "deepset-ai/a-model-name"): - """ - Haystack nodes should have an `__init__` method where they define: - - - `self.inputs = []`: - A list with all the edges they can possibly receive input from - - - `self.outputs = []`: - A list with the edges they might possibly produce as output - - - `self.init_parameters = {}`: - Any state they wish to be persisted in their YAML serialization. - These values will be given to the `__init__` method of a new instance - when the pipeline is deserialized. - - The `__init__` must be extremely lightweight, because it's a frequent - operation during the construction and validation of the pipeline. If a node - has some heavy state to initialize (models, backends, etc...) refer to the - `warm_up()` method. - """ - # Lightweight state can be initialized here, for example storing the model name - # to be loaded later. See self.warm_up() - self.model = None - self.model_name = model_name - self.how_many_times_have_I_been_called = 0 + def __init__(self, model_name: str: "deepset-ai/a-model-name" - # Contract - all three are mandatory. - self.init_parameters = {"model_name": model_name} - self.inputs = ["input_name"] - self.outputs = ["output_name"] + ): + """ + Haystack nodes should have an `__init__` method where they define: - def warm_up(self): - """ - Optional method. + - `self.inputs = []`: + A list with all the edges they can possibly receive input from - This method is called by Pipeline before the graph execution. - Make sure to avoid double-initializations, because Pipeline will not keep - track of which nodes it called `warm_up` on. - """ - if not self.model: - self.model = AutoModel.load_from_pretrained(self.model_name) + - `self.outputs = []`: + A list with the edges they might possibly produce as output - def run( + - `self.init_parameters = {}`: + Any state they wish to be persisted in their YAML serialization. + These values will be given to the `__init__` method of a new instance + when the pipeline is deserialized. + + The `__init__` must be extremely lightweight, because it's a frequent + operation during the construction and validation of the pipeline. If a node + has some heavy state to initialize (models, backends, etc...) refer to the + `warm_up()` method. + """ + # Lightweight state can be initialized here, for example storing the model name + # to be loaded later. See self.warm_up() + self.embeddings_model = None + self.model_name = model_name + self.how_many_times_have_I_been_called = 0 + + # Contract - all three are mandatory. + self.init_parameters = {"model_name": model_name} + self.inputs = ["input_name"] + self.outputs = ["output_name"] + + +def warm_up(self): + """ + Optional method. + + This method is called by Pipeline before the graph execution. + Make sure to avoid double-initializations, because Pipeline will not keep + track of which nodes it called `warm_up` on. + """ + if not self.embeddings_model: + self.embeddings_model = AutoModel.load_from_pretrained(self.model_name) + + +def run( self, name: str, data: List[Tuple[str, Any]], parameters: Dict[str, Any], stores: Dict[str, Any], - ): - """ - Mandatory method. - - This is the method where the main functionality of the node should be carried out. - It's called by `Pipeline.run()`, which passes the following parameters to it: - - - `name: str`: the name of the node. Allows the node to find its own parameters in the `parameters` dictionary (see below). - - - `data: List[Tuple[str, Any]]`: the input data. - Pipeline guarantees that the following assert always passes: `assert self.inputs == [name for name, value in data]`, - which means that: - - `data` is of the same length as `self.inputs`. - - `data` contains one tuple for each string stored in `self.inputs`. - - no guarantee is given on the values of these tuples: notably, if there was a decision node upstream, some values might be `None`. - For example, if a node declares `self.inputs = ["value", "value"]` (think of a Sum node), `data` might look like: - - `[("value", 1), ("value", 10)]` - - `[("value", None), ("value", 10)]` - - `[("value", None), ("value", None)]`, or even - - `[("value", 1), ("value", ["something", "unexpected"])]` - but it will never look like: - - `[("value", 1), ("value", 10), ("value", 100)]`, - - `[("value": 15)]` or - - `[("value": 15), ("unexpected", 10)]`. - - - `parameters: Dict[str, Dict[str, Any]]`: a dictionary of dictionaries with all the parameters for all nodes. - Note that all nodes have access to all parameters for all other nodes: this might come handy to nodes like `Agent`s, that - want to influence the behavior of nodes downstream. - Nodes can access their own parameters using `name`, but they must not assume their name is present in the dictionary. - Therefore the best way to get the parameters is with `my_parameters = parameters.get(name, {})` - - - `stores`: a dictionary of all the (Document)Stores connected to this pipeline. - - Pipeline expect the output of this function to be a tuple in the following format: - - `( {edge: value for edge in }, {the parameters dictionary}) - - Which means that: - - Nodes are not forced to produce output on all the expected outputs: for example nodes taking a decision, like classifiers, - can produce output on a subset of the expected output edges and Pipeline will figure out the rest. - - Nodes must not add any key in the data dictionary that is not present in `self.outputs`, - - Nodes can alter the content of `parameters` and their changes will be propagated downstream. - """ - self.how_many_times_have_I_been_called += 1 - - value = data[0][1] - print(f"Hello I'm {name}! This instance have been called {self.how_many_times_have_I_been_called} times and this is the value I received: {value}") +): + """ + Mandatory method. + + This is the method where the main functionality of the node should be carried out. + It's called by `Pipeline.run()`, which passes the following parameters to it: + + - `name: str`: the name of the node. Allows the node to find its own parameters in the `parameters` dictionary (see below). + + - `data: List[Tuple[str, Any]]`: the input data. + Pipeline guarantees that the following assert always passes: `assert self.inputs == [name for name, value in data]`, + which means that: + - `data` is of the same length as `self.inputs`. + - `data` contains one tuple for each string stored in `self.inputs`. + - no guarantee is given on the values of these tuples: notably, if there was a decision node upstream, some values might be `None`. + For example, if a node declares `self.inputs = ["value", "value"]` (think of a Sum node), `data` might look like: + - `[("value", 1), ("value", 10)]` + - `[("value", None), ("value", 10)]` + - `[("value", None), ("value", None)]`, or even + - `[("value", 1), ("value", ["something", "unexpected"])]` + but it will never look like: + - `[("value", 1), ("value", 10), ("value", 100)]`, + - `[("value": 15)]` or + - `[("value": 15), ("unexpected", 10)]`. + + - `parameters: Dict[str, Dict[str, Any]]`: a dictionary of dictionaries with all the parameters for all nodes. + Note that all nodes have access to all parameters for all other nodes: this might come handy to nodes like `Agent`s, that + want to influence the behavior of nodes downstream. + Nodes can access their own parameters using `name`, but they must not assume their name is present in the dictionary. + Therefore the best way to get the parameters is with `my_parameters = parameters.get(name, {})` + + - `stores`: a dictionary of all the (Document)Stores connected to this pipeline. + + Pipeline expect the output of this function to be a tuple in the following format: + + `( {edge: value for edge in }, {the parameters dictionary}) + + Which means that: + - Nodes are not forced to produce output on all the expected outputs: for example nodes taking a decision, like classifiers, + can produce output on a subset of the expected output edges and Pipeline will figure out the rest. + - Nodes must not add any key in the data dictionary that is not present in `self.outputs`, + - Nodes can alter the content of `parameters` and their changes will be propagated downstream. + """ + self.how_many_times_have_I_been_called += 1 - return ({self.outputs[0]: value}, parameters) + value = data[0][1] + print( + f"Hello I'm {name}! This instance have been called {self.how_many_times_have_I_been_called} times and this is the value I received: {value}") + + return ({self.outputs[0]: value}, parameters) ``` This contract is stored in the docstring of `@node` and acts as the single source of truth. diff --git a/test/components/embedders/test_sentence_transformers_embedding_backend.py b/test/components/embedders/test_sentence_transformers_embedding_backend.py index cbdc812ac7..a10acf9200 100644 --- a/test/components/embedders/test_sentence_transformers_embedding_backend.py +++ b/test/components/embedders/test_sentence_transformers_embedding_backend.py @@ -39,4 +39,4 @@ def test_embedding_function_with_kwargs(mock_sentence_transformer): data = ["sentence1", "sentence2"] embedding_backend.embed(data=data, normalize_embeddings=True) - embedding_backend.model.encode.assert_called_once_with(data, normalize_embeddings=True) + embedding_backend.embeddings_model.encode.assert_called_once_with(data, normalize_embeddings=True) From 47bf8ce20846ed29ae16c1faa8c2248abb144036 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Thu, 25 Apr 2024 16:12:04 +0200 Subject: [PATCH 05/13] cleaning up tests --- e2e/pipelines/test_evaluation_pipeline.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py index 5740297215..196a429fbf 100644 --- a/e2e/pipelines/test_evaluation_pipeline.py +++ b/e2e/pipelines/test_evaluation_pipeline.py @@ -122,15 +122,8 @@ def run_rag_pipeline(documents, evaluation_questions, rag_pipeline_a): ) def test_evaluation_pipeline(samples_path): """ - Test the evaluation pipeline + Test custom built evaluation pipeline """ - docs = [] - articles = os.listdir(str(samples_path) + "/test_documents/") - for article in articles: - with open(f"{str(samples_path)}/test_documents/{article}", "r") as f: - for text in f.read().split("\n"): - docs.append(Document(content=text, meta={"name": article})) - doc_store = indexing_pipeline(docs) eval_questions = [ { @@ -150,6 +143,15 @@ def test_evaluation_pipeline(samples_path): "ground_truth_doc": ["Materialism.txt"], }, ] + + docs = [] + full_path = os.path.join(str(samples_path) + "/test_documents/") + for article in os.listdir(full_path): + with open(f"{full_path}/{article}", "r") as f: + for text in f.read().split("\n"): + docs.append(Document(content=text, meta={"name": article})) + doc_store = indexing_pipeline(docs) + questions = [q["question"] for q in eval_questions] truth_answers = [q["answer"] for q in eval_questions] @@ -195,7 +197,6 @@ def test_evaluation_pipeline(samples_path): df_score_report = evaluation_result_a.score_report() assert len(df_score_report) == 6 - print(df_score_report.columns) assert list(df_score_report.columns) == ["score"] assert list(df_score_report.index) == [ "Mean Reciprocal Rank", From 268e094a96f8e88231ef1be5fbccc477dd8305c4 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Thu, 25 Apr 2024 16:22:31 +0200 Subject: [PATCH 06/13] cleaning up tests --- e2e/pipelines/test_evaluation_pipeline.py | 26 +++++++-------- e2e/samples/test_documents/Strasbourg.txt | 34 -------------------- e2e/samples/test_documents/War_on_Terror.txt | 31 ------------------ 3 files changed, 13 insertions(+), 78 deletions(-) delete mode 100644 e2e/samples/test_documents/Strasbourg.txt delete mode 100644 e2e/samples/test_documents/War_on_Terror.txt diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py index 196a429fbf..883b45c88f 100644 --- a/e2e/pipelines/test_evaluation_pipeline.py +++ b/e2e/pipelines/test_evaluation_pipeline.py @@ -50,20 +50,20 @@ def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int): Question: {{question}} Answer: """ - rag_pipeline = Pipeline() - rag_pipeline.add_component("embedder", SentenceTransformersTextEmbedder(model=embeddings_model)) - rag_pipeline.add_component("retriever", InMemoryEmbeddingRetriever(document_store, top_k=top_k)) - rag_pipeline.add_component("prompt_builder", PromptBuilder(template=template)) - rag_pipeline.add_component("generator", OpenAIGenerator(model="gpt-3.5-turbo")) - rag_pipeline.add_component("answer_builder", AnswerBuilder()) - rag_pipeline.connect("embedder", "retriever.query_embedding") - rag_pipeline.connect("retriever", "prompt_builder.documents") - rag_pipeline.connect("prompt_builder", "generator") - rag_pipeline.connect("generator.replies", "answer_builder.replies") - rag_pipeline.connect("generator.meta", "answer_builder.meta") - rag_pipeline.connect("retriever", "answer_builder.documents") + rag = Pipeline() + rag.add_component("embedder", SentenceTransformersTextEmbedder(model=embeddings_model)) + rag.add_component("retriever", InMemoryEmbeddingRetriever(document_store, top_k=top_k)) + rag.add_component("prompt_builder", PromptBuilder(template=template)) + rag.add_component("generator", OpenAIGenerator(model="gpt-3.5-turbo")) + rag.add_component("answer_builder", AnswerBuilder()) + rag.connect("embedder", "retriever.query_embedding") + rag.connect("retriever", "prompt_builder.documents") + rag.connect("prompt_builder", "generator") + rag.connect("generator.replies", "answer_builder.replies") + rag.connect("generator.meta", "answer_builder.meta") + rag.connect("retriever", "answer_builder.documents") - return rag_pipeline + return rag def evaluation_pipeline(questions, truth_docs, truth_answers, retrieved_docs, contexts, pred_answers): diff --git a/e2e/samples/test_documents/Strasbourg.txt b/e2e/samples/test_documents/Strasbourg.txt deleted file mode 100644 index bf64384234..0000000000 --- a/e2e/samples/test_documents/Strasbourg.txt +++ /dev/null @@ -1,34 +0,0 @@ -Strasbourg (/ˈstræzbɜːrɡ/, French pronunciation: ​[stʁaz.buʁ, stʁas.buʁ]; Alsatian: Strossburi; German: Straßburg, [ˈʃtʁaːsbʊɐ̯k]) is the capital and largest city of the Alsace-Champagne-Ardenne-Lorraine (ACAL) region in eastern France and is the official seat of the European Parliament. Located close to the border with Germany, it is the capital of the Bas-Rhin département. The city and the region of Alsace were historically predominantly Alemannic-speaking, hence the city's Germanic name. In 2013, the city proper had 275,718 inhabitants, Eurométropole de Strasbourg (Greater Strasbourg) had 475,934 inhabitants and the Arrondissement of Strasbourg had 482,384 inhabitants. With a population of 768,868 in 2012, Strasbourg's metropolitan area (only the part of the metropolitan area on French territory) is the ninth largest in France and home to 13% of the ACAL region's inhabitants. The transnational Eurodistrict Strasbourg-Ortenau had a population of 915,000 inhabitants in 2014. -Strasbourg's historic city centre, the Grande Île (Grand Island), was classified a World Heritage site by UNESCO in 1988, the first time such an honour was placed on an entire city centre. Strasbourg is immersed in the Franco-German culture and although violently disputed throughout history, has been a bridge of unity between France and Germany for centuries, especially through the University of Strasbourg, currently the second largest in France, and the coexistence of Catholic and Protestant culture. The largest Islamic place of worship in France, the Strasbourg Grand Mosque, was inaugurated by French Interior Minister Manuel Valls on 27 September 2012. -Strasbourg is situated on the eastern border of France with Germany. This border is formed by the River Rhine, which also forms the eastern border of the modern city, facing across the river to the German town Kehl. The historic core of Strasbourg however lies on the Grande Île in the River Ill, which here flows parallel to, and roughly 4 kilometres (2.5 mi) from, the Rhine. The natural courses of the two rivers eventually join some distance downstream of Strasbourg, although several artificial waterways now connect them within the city. -The Romans under Nero Claudius Drusus established a military outpost belonging to the Germania Superior Roman province at Strasbourg's current location, and named it Argentoratum. (Hence the town is commonly called Argentina in medieval Latin.) The name "Argentoratum" was first mentioned in 12 BC and the city celebrated its 2,000th birthday in 1988. "Argentorate" as the toponym of the Gaulish settlement preceded it before being Latinized, but it is not known by how long. The Roman camp was destroyed by fire and rebuilt six times between the first and the fifth centuries AD: in 70, 97, 235, 355, in the last quarter of the fourth century, and in the early years of the fifth century. It was under Trajan and after the fire of 97 that Argentoratum received its most extended and fortified shape. From the year 90 on, the Legio VIII Augusta was permanently stationed in the Roman camp of Argentoratum. It then included a cavalry section and covered an area of approximately 20 hectares. Other Roman legions temporarily stationed in Argentoratum were the Legio XIV Gemina and the Legio XXI Rapax, the latter during the reign of Nero. -The centre of Argentoratum proper was situated on the Grande Île (Cardo: current Rue du Dôme, Decumanus: current Rue des Hallebardes). The outline of the Roman "castrum" is visible in the street pattern in the Grande Ile. Many Roman artifacts have also been found along the current Route des Romains, the road that led to Argentoratum, in the suburb of Kœnigshoffen. This was where the largest burial places were situated, as well as the densest concentration of civilian dwelling places and commerces next to the camp. Among the most outstanding finds in Kœnigshoffen were (found in 1911–12) the fragments of a grand Mithraeum that had been shattered by early Christians in the fourth century. From the fourth century, Strasbourg was the seat of the Bishopric of Strasbourg (made an Archbishopric in 1988). Archaeological excavations below the current Église Saint-Étienne in 1948 and 1956 unearthed the apse of a church dating back to the late fourth or early fifth century, considered to be the oldest church in Alsace. It is supposed that this was the first seat of the Roman Catholic Diocese of Strasbourg. -In the fifth century Strasbourg was occupied successively by Alemanni, Huns, and Franks. In the ninth century it was commonly known as Strazburg in the local language, as documented in 842 by the Oaths of Strasbourg. This trilingual text contains, alongside texts in Latin and Old High German (teudisca lingua), the oldest written variety of Gallo-Romance (lingua romana) clearly distinct from Latin, the ancestor of Old French. The town was also called Stratisburgum or Strateburgus in Latin, from which later came Strossburi in Alsatian and Straßburg in Standard German, and then Strasbourg in French. The Oaths of Strasbourg is considered as marking the birth of the two countries of France and Germany with the division of the Carolingian Empire. -A revolution in 1332 resulted in a broad-based city government with participation of the guilds, and Strasbourg declared itself a free republic. The deadly bubonic plague of 1348 was followed on 14 February 1349 by one of the first and worst pogroms in pre-modern history: over a thousand Jews were publicly burnt to death, with the remainder of the Jewish population being expelled from the city. Until the end of the 18th century, Jews were forbidden to remain in town after 10 pm. The time to leave the city was signalled by a municipal herald blowing the Grüselhorn (see below, Museums, Musée historique);. A special tax, the Pflastergeld (pavement money), was furthermore to be paid for any horse that a Jew would ride or bring into the city while allowed to. -In the 1520s during the Protestant Reformation, the city, under the political guidance of Jacob Sturm von Sturmeck and the spiritual guidance of Martin Bucer embraced the religious teachings of Martin Luther. Their adherents established a Gymnasium, headed by Johannes Sturm, made into a University in the following century. The city first followed the Tetrapolitan Confession, and then the Augsburg Confession. Protestant iconoclasm caused much destruction to churches and cloisters, notwithstanding that Luther himself opposed such a practice. Strasbourg was a centre of humanist scholarship and early book-printing in the Holy Roman Empire, and its intellectual and political influence contributed much to the establishment of Protestantism as an accepted denomination in the southwest of Germany. (John Calvin spent several years as a political refugee in the city). The Strasbourg Councillor Sturm and guildmaster Matthias represented the city at the Imperial Diet of Speyer (1529), where their protest led to the schism of the Catholic Church and the evolution of Protestantism. Together with four other free cities, Strasbourg presented the confessio tetrapolitana as its Protestant book of faith at the Imperial Diet of Augsburg in 1530, where the slightly different Augsburg Confession was also handed over to Charles V, Holy Roman Emperor. -Louis' advisors believed that, as long as Strasbourg remained independent, it would endanger the King's newly annexed territories in Alsace, and, that to defend these large rural lands effectively, a garrison had to be placed in towns such as Strasbourg. Indeed, the bridge over the Rhine at Strasbourg had been used repeatedly by Imperial (Holy Roman Empire) forces, and three times during the Franco-Dutch War Strasbourg had served as a gateway for Imperial invasions into Alsace. In September 1681 Louis' forces, though lacking a clear casus belli, surrounded the city with overwhelming force. After some negotiation, Louis marched into the city unopposed on 30 September 1681 and proclaimed its annexation. -This annexation was one of the direct causes of the brief and bloody War of the Reunions whose outcome left the French in possession. The French annexation was recognized by the Treaty of Ryswick (1697). The official policy of religious intolerance which drove most Protestants from France after the revocation of the Edict of Nantes in 1685 was not applied in Strasbourg and in Alsace, because both had a special status as a province à l'instar de l'étranger effectif (a kind of foreign province of the king of France). Strasbourg Cathedral, however, was taken from the Lutherans to be returned to the Catholics as the French authorities tried to promote Catholicism wherever they could (some other historic churches remained in Protestant hands). Its language also remained overwhelmingly German: the German Lutheran university persisted until the French Revolution. Famous students included Goethe and Herder. -Strasbourg's status as a free city was revoked by the French Revolution. Enragés, most notoriously Eulogius Schneider, ruled the city with an increasingly iron hand. During this time, many churches and monasteries were either destroyed or severely damaged. The cathedral lost hundreds of its statues (later replaced by copies in the 19th century) and in April 1794, there was talk of tearing its spire down, on the grounds that it was against the principle of equality. The tower was saved, however, when in May of the same year citizens of Strasbourg crowned it with a giant tin Phrygian cap. This artifact was later kept in the historical collections of the city until it was destroyed by the Germans in 1870 during the Franco-Prussian war. -During the Franco-Prussian War and the Siege of Strasbourg, the city was heavily bombarded by the Prussian army. The bombardment of the city was meant to break the morale of the people of Strasbourg. On 24 and 26 August 1870, the Museum of Fine Arts was destroyed by fire, as was the Municipal Library housed in the Gothic former Dominican church, with its unique collection of medieval manuscripts (most famously the Hortus deliciarum), rare Renaissance books, archeological finds and historical artifacts. The gothic cathedral was damaged as well as the medieval church of Temple Neuf, the theatre, the city hall, the court of justice and many houses. At the end of the siege 10,000 inhabitants were left without shelter; over 600 died, including 261 civilians, and 3200 were injured, including 1,100 civilians. -In 1871, after the end of the war, the city was annexed to the newly established German Empire as part of the Reichsland Elsass-Lothringen under the terms of the Treaty of Frankfurt. As part of Imperial Germany, Strasbourg was rebuilt and developed on a grand and representative scale, such as the Neue Stadt, or "new city" around the present Place de la République. Historian Rodolphe Reuss and Art historian Wilhelm von Bode were in charge of rebuilding the municipal archives, libraries and museums. The University, founded in 1567 and suppressed during the French Revolution as a stronghold of German sentiment,[citation needed] was reopened in 1872 under the name Kaiser-Wilhelms-Universität. -A belt of massive fortifications was established around the city, most of which still stands today, renamed after French generals and generally classified as Monuments historiques; most notably Fort Roon (now Fort Desaix) and Fort Podbielski (now Fort Ducrot) in Mundolsheim, Fort von Moltke (now Fort Rapp) in Reichstett, Fort Bismarck (now Fort Kléber) in Wolfisheim, Fort Kronprinz (now Fort Foch) in Niederhausbergen, Fort Kronprinz von Sachsen (now Fort Joffre) in Holtzheim and Fort Großherzog von Baden (now Fort Frère) in Oberhausbergen. -Following the defeat of the German empire in World War I and the abdication of the German Emperor, some revolutionary insurgents declared Alsace-Lorraine as an independent Republic, without preliminary referendum or vote. On 11 November 1918 (Armistice Day), communist insurgents proclaimed a "soviet government" in Strasbourg, following the example of Kurt Eisner in Munich as well as other German towns. French troops commanded by French general Henri Gouraud entered triumphantly in the city on 22 November. A major street of the city now bears the name of that date (Rue du 22 Novembre) which celebrates the entry of the French in the city. Viewing the massive cheering crowd gathered under the balcony of Strasbourg's town hall, French President Raymond Poincaré stated that "the plebiscite is done". -In 1919, following the Treaty of Versailles, the city was restituted to France in accordance with U.S. President Woodrow Wilson's "Fourteen Points" without a referendum. The date of the assignment was retroactively established on Armistice Day. It is doubtful whether a referendum in Strasbourg would have ended in France's favour since the political parties striving for an autonomous Alsace or a connection to France accounted only for a small proportion of votes in the last Reichstag as well as in the local elections. The Alsatian autonomists who were pro French had won many votes in the more rural parts of the region and other towns since the annexation of the region by Germany in 1871. The movement started with the first election for the Reichstag; those elected were called "les députés protestataires", and until the fall of Bismarck in 1890, they were the only deputies elected by the Alsatians to the German parliament demanding the return of those territories to France. At the last Reichstag election in Strasbourg and its periphery, the clear winners were the Social Democrats; the city was the administrative capital of the region, was inhabited by many Germans appointed by the central government in Berlin and its flourishing economy attracted many Germans. This could explain the difference between the rural vote and the one in Strasbourg. After the war, many Germans left Strasbourg and went back to Germany; some of them were denounced by the locals or expelled by the newly appointed authorities. The Saverne Affair was vivid in the memory among the Alsatians. -Between the German invasion of Poland on 1 September 1939 and the Anglo-French declaration of War against the German Reich on 3 September 1939, the entire city (a total of 120,000 people) was evacuated, like other border towns as well. Until the arrival of the Wehrmacht troops mid-June 1940, the city was, for ten months, completely empty, with the exception of the garrisoned soldiers. The Jews of Strasbourg had been evacuated to Périgueux and Limoges, the University had been evacuated to Clermont-Ferrand. -After the ceasefire following the Fall of France in June 1940, Alsace was annexed to Germany and a rigorous policy of Germanisation was imposed upon it by the Gauleiter Robert Heinrich Wagner. When, in July 1940, the first evacuees were allowed to return, only residents of Alsatian origin were admitted. The last Jews were deported on 15 July 1940 and the main synagogue, a huge Romanesque revival building that had been a major architectural landmark with its 54-metre-high dome since its completion in 1897, was set ablaze, then razed. -In September 1940 the first Alsatian resistance movement led by Marcel Weinum called La main noire (The black hand) was created. It was composed by a group of 25 young men aged from 14 to 18 years old who led several attacks against the German occupation. The actions culminated with the attack of the Gauleiter Robert Wagner, the highest commander of Alsace directly under the order of Hitler. In March 1942, Marcel Weinum was prosecuted by the Gestapo and sentenced to be beheaded at the age of 18 in April 1942 in Stuttgart, Germany. His last words will be: "If I have to die, I shall die but with a pure heart". From 1943 the city was bombarded by Allied aircraft. While the First World War had not notably damaged the city, Anglo-American bombing caused extensive destruction in raids of which at least one was allegedly carried out by mistake. In August 1944, several buildings in the Old Town were damaged by bombs, particularly the Palais Rohan, the Old Customs House (Ancienne Douane) and the Cathedral. On 23 November 1944, the city was officially liberated by the 2nd French Armoured Division under General Leclerc. He achieved the oath that he made with his soldiers, after the decisive Capture of Kufra. With the Oath of Kuffra, they swore to keep up the fight until the French flag flew over the Cathedral of Strasbourg. -Many people from Strasbourg were incorporated in the German Army against their will, and were sent to the eastern front, those young men and women were called Malgré-nous. Many tried to escape from the incorporation, join the French Resistance, or desert the Wehrmacht but many couldn't because they were running the risk of having their families sent to work or concentration camps by the Germans. Many of these men, especially those who did not answer the call immediately, were pressured to "volunteer" for service with the SS, often by direct threats on their families. This threat obliged the majority of them to remain in the German army. After the war, the few that survived were often accused of being traitors or collaborationists, because this tough situation was not known in the rest of France, and they had to face the incomprehension of many. In July 1944, 1500 malgré-nous were released from Soviet captivity and sent to Algiers, where they joined the Free French Forces. Nowadays history recognizes the suffering of those people, and museums, public discussions and memorials have been built to commemorate this terrible period of history of this part of Eastern France (Alsace and Moselle). Liberation of Strasbourg took place on 23 November 1944. -In 1949, the city was chosen to be the seat of the Council of Europe with its European Court of Human Rights and European Pharmacopoeia. Since 1952, the European Parliament has met in Strasbourg, which was formally designated its official 'seat' at the Edinburgh meeting of the European Council of EU heads of state and government in December 1992. (This position was reconfirmed and given treaty status in the 1997 Treaty of Amsterdam). However, only the (four-day) plenary sessions of the Parliament are held in Strasbourg each month, with all other business being conducted in Brussels and Luxembourg. Those sessions take place in the Immeuble Louise Weiss, inaugurated in 1999, which houses the largest parliamentary assembly room in Europe and of any democratic institution in the world. Before that, the EP sessions had to take place in the main Council of Europe building, the Palace of Europe, whose unusual inner architecture had become a familiar sight to European TV audiences. In 1992, Strasbourg became the seat of the Franco-German TV channel and movie-production society Arte. -In addition to the cathedral, Strasbourg houses several other medieval churches that have survived the many wars and destructions that have plagued the city: the Romanesque Église Saint-Étienne, partly destroyed in 1944 by Allied bombing raids, the part Romanesque, part Gothic, very large Église Saint-Thomas with its Silbermann organ on which Wolfgang Amadeus Mozart and Albert Schweitzer played, the Gothic Église protestante Saint-Pierre-le-Jeune with its crypt dating back to the seventh century and its cloister partly from the eleventh century, the Gothic Église Saint-Guillaume with its fine early-Renaissance stained glass and furniture, the Gothic Église Saint-Jean, the part Gothic, part Art Nouveau Église Sainte-Madeleine, etc. The Neo-Gothic church Saint-Pierre-le-Vieux Catholique (there is also an adjacent church Saint-Pierre-le-Vieux Protestant) serves as a shrine for several 15th-century wood worked and painted altars coming from other, now destroyed churches and installed there for public display. Among the numerous secular medieval buildings, the monumental Ancienne Douane (old custom-house) stands out. -The German Renaissance has bequeathed the city some noteworthy buildings (especially the current Chambre de commerce et d'industrie, former town hall, on Place Gutenberg), as did the French Baroque and Classicism with several hôtels particuliers (i.e. palaces), among which the Palais Rohan (1742, now housing three museums) is the most spectacular. Other buildings of its kind are the "Hôtel de Hanau" (1736, now the city hall), the Hôtel de Klinglin (1736, now residence of the préfet), the Hôtel des Deux-Ponts (1755, now residence of the military governor), the Hôtel d'Andlau-Klinglin (1725, now seat of the administration of the Port autonome de Strasbourg) etc. The largest baroque building of Strasbourg though is the 150 m (490 ft) long 1720s main building of the Hôpital civil. As for French Neo-classicism, it is the Opera House on Place Broglie that most prestigiously represents this style. -Strasbourg also offers high-class eclecticist buildings in its very extended German district, the Neustadt, being the main memory of Wilhelmian architecture since most of the major cities in Germany proper suffered intensive damage during World War II. Streets, boulevards and avenues are homogeneous, surprisingly high (up to seven stories) and broad examples of German urban lay-out and of this architectural style that summons and mixes up five centuries of European architecture as well as Neo-Egyptian, Neo-Greek and Neo-Babylonian styles. The former imperial palace Palais du Rhin, the most political and thus heavily criticized of all German Strasbourg buildings epitomizes the grand scale and stylistic sturdiness of this period. But the two most handsome and ornate buildings of these times are the École internationale des Pontonniers (the former Höhere Mädchenschule, girls college) with its towers, turrets and multiple round and square angles and the École des Arts décoratifs with its lavishly ornate façade of painted bricks, woodwork and majolica. -As for modern and contemporary architecture, Strasbourg possesses some fine Art Nouveau buildings (such as the huge Palais des Fêtes and houses and villas like Villa Schutzenberger and Hôtel Brion), good examples of post-World War II functional architecture (the Cité Rotterdam, for which Le Corbusier did not succeed in the architectural contest) and, in the very extended Quartier Européen, some spectacular administrative buildings of sometimes utterly large size, among which the European Court of Human Rights building by Richard Rogers is arguably the finest. Other noticeable contemporary buildings are the new Music school Cité de la Musique et de la Danse, the Musée d'Art moderne et contemporain and the Hôtel du Département facing it, as well as, in the outskirts, the tramway-station Hoenheim-Nord designed by Zaha Hadid. -Strasbourg features a number of prominent parks, of which several are of cultural and historical interest: the Parc de l'Orangerie, laid out as a French garden by André le Nôtre and remodeled as an English garden on behalf of Joséphine de Beauharnais, now displaying noteworthy French gardens, a neo-classical castle and a small zoo; the Parc de la Citadelle, built around impressive remains of the 17th-century fortress erected close to the Rhine by Vauban; the Parc de Pourtalès, laid out in English style around a baroque castle (heavily restored in the 19th century) that now houses a small three-star hotel, and featuring an open-air museum of international contemporary sculpture. The Jardin botanique de l'Université de Strasbourg (botanical garden) was created under the German administration next to the Observatory of Strasbourg, built in 1881, and still owns some greenhouses of those times. The Parc des Contades, although the oldest park of the city, was completely remodeled after World War II. The futuristic Parc des Poteries is an example of European park-conception in the late 1990s. The Jardin des deux Rives, spread over Strasbourg and Kehl on both sides of the Rhine opened in 2004 and is the most extended (60-hectare) park of the agglomeration. The most recent park is Parc du Heyritz (8,7 ha), opened in 2014 along a canal facing the hôpital civil. -Unlike most other cities, Strasbourg's collections of European art are divided into several museums according not only to type and area, but also to epoch. Old master paintings from the Germanic Rhenish territories and until 1681 are displayed in the Musée de l'Œuvre Notre-Dame, old master paintings from all the rest of Europe (including the Dutch Rhenish territories) and until 1871 as well as old master paintings from the Germanic Rhenish territories between 1681 and 1871 are displayed in the Musée des Beaux-Arts. Old master graphic arts until 1871 is displayed in the Cabinet des estampes et dessins. Decorative arts until 1681 ("German period") are displayed in the Musée de l'Œuvre Notre-Dame, decorative arts from 1681 to 1871 ("French period") are displayed in the Musée des Arts décoratifs. International art (painting, sculpture, graphic arts) and decorative art since 1871 is displayed in the Musée d'art moderne et contemporain. The latter museum also displays the city's photographic library. -Strasbourg, well known as centre of humanism, has a long history of excellence in higher-education, at the crossroads of French and German intellectual traditions. Although Strasbourg had been annexed by the Kingdom of France in 1683, it still remained connected to the German-speaking intellectual world throughout the 18th century and the university attracted numerous students from the Holy Roman Empire, including Goethe, Metternich and Montgelas, who studied law in Strasbourg, among the most prominent. Nowadays, Strasbourg is known to offer among the best university courses in France, after Paris. -The Bibliothèque nationale et universitaire (BNU) is, with its collection of more than 3,000,000 titles, the second largest library in France after the Bibliothèque nationale de France. It was founded by the German administration after the complete destruction of the previous municipal library in 1871 and holds the unique status of being simultaneously a students' and a national library. The Strasbourg municipal library had been marked erroneously as "City Hall" in a French commercial map, which had been captured and used by the German artillery to lay their guns. A librarian from Munich later pointed out "...that the destruction of the precious collection was not the fault of a German artillery officer, who used the French map, but of the slovenly and inaccurate scholarship of a Frenchman." -As one of the earliest centers of book-printing in Europe (see above: History), Strasbourg for a long time held a large number of incunabula—documents printed before 1500—in her library as one of her most precious heritages. After the total destruction of this institution in 1870, however, a new collection had to be reassembled from scratch. Today, Strasbourg's different public and institutional libraries again display a sizable total number of incunabula, distributed as follows: Bibliothèque nationale et universitaire, ca. 2 098 Médiathèque de la ville et de la communauté urbaine de Strasbourg, 394 Bibliothèque du Grand Séminaire, 238 Médiathèque protestante, 94 and Bibliothèque alsatique du Crédit Mutuel, 5. -City transportation in Strasbourg includes the futurist-looking Strasbourg tramway that opened in 1994 and is operated by the regional transit company Compagnie des Transports Strasbourgeois (CTS), consisting of 6 lines with a total length of 55.8 km (34.7 mi). The CTS also operates a comprehensive bus network throughout the city that is integrated with the trams. With more than 500 km (311 mi) of bicycle paths, biking in the city is convenient and the CTS operates a cheap bike-sharing scheme named Vélhop'. The CTS, and its predecessors, also operated a previous generation of tram system between 1878 and 1960, complemented by trolleybus routes between 1939 and 1962. -Being a city on the Ill and close to the Rhine, Strasbourg has always been an important centre of fluvial navigation, as is attested by archeological findings. In 1682 the Canal de la Bruche was added to the river navigations, initially to provide transport for sandstone from quarries in the Vosges for use in the fortification of the city. That canal has since closed, but the subsequent Canal du Rhone au Rhine, Canal de la Marne au Rhin and Grand Canal d'Alsace are still in use, as is the important activity of the Port autonome de Strasbourg. Water tourism inside the city proper attracts hundreds of thousands of tourists yearly. -The tram system that now criss-crosses the historic city centre complements walking and biking in it. The centre has been transformed into a pedestrian priority zone that enables and invites walking and biking by making these active modes of transport comfortable, safe and enjoyable. These attributes are accomplished by applying the principle of "filtered permeability" to the existing irregular network of streets. It means that the network adaptations favour active transportation and, selectively, "filter out" the car by reducing the number of streets that run through the centre. While certain streets are discontinuous for cars, they connect to a network of pedestrian and bike paths which permeate the entire centre. In addition, these paths go through public squares and open spaces increasing the enjoyment of the trip. This logic of filtering a mode of transport is fully expressed in a comprehensive model for laying out neighbourhoods and districts – the Fused Grid. -At present the A35 autoroute, which parallels the Rhine between Karlsruhe and Basel, and the A4 autoroute, which links Paris with Strasbourg, penetrate close to the centre of the city. The Grand contournement ouest (GCO) project, programmed since 1999, plans to construct a 24 km (15 mi) long highway connection between the junctions of the A4 and the A35 autoroutes in the north and of the A35 and A352 autoroutes in the south. This routes well to the west of the city and is meant to divest a significant portion of motorized traffic from the unité urbaine. diff --git a/e2e/samples/test_documents/War_on_Terror.txt b/e2e/samples/test_documents/War_on_Terror.txt deleted file mode 100644 index 1601e747ba..0000000000 --- a/e2e/samples/test_documents/War_on_Terror.txt +++ /dev/null @@ -1,31 +0,0 @@ -On 16 September 2001, at Camp David, President George W. Bush used the phrase war on terrorism in an unscripted and controversial comment when he said, "This crusade – this war on terrorism – is going to take a while, ... " Bush later apologized for this remark due to the negative connotations the term crusade has to people, e.g. of Muslim faith. The word crusade was not used again. On 20 September 2001, during a televised address to a joint session of congress, Bush stated that, "(o)ur 'war on terror' begins with al-Qaeda, but it does not end there. It will not end until every terrorist group of global reach has been found, stopped, and defeated." -U.S. President Barack Obama has rarely used the term, but in his inaugural address on 20 January 2009, he stated "Our nation is at war, against a far-reaching network of violence and hatred." In March 2009 the Defense Department officially changed the name of operations from "Global War on Terror" to "Overseas Contingency Operation" (OCO). In March 2009, the Obama administration requested that Pentagon staff members avoid use of the term, instead using "Overseas Contingency Operation". Basic objectives of the Bush administration "war on terror", such as targeting al Qaeda and building international counterterrorism alliances, remain in place. In December 2012, Jeh Johnson, the General Counsel of the Department of Defense, stated that the military fight will be replaced by a law enforcement operation when speaking at Oxford University, predicting that al Qaeda will be so weakened to be ineffective, and has been "effectively destroyed", and thus the conflict will not be an armed conflict under international law. In May 2013, Obama stated that the goal is "to dismantle specific networks of violent extremists that threaten America"; which coincided with the U.S. Office of Management and Budget having changed the wording from "Overseas Contingency Operations" to "Countering Violent Extremism" in 2010. -Because the actions involved in the "war on terrorism" are diffuse, and the criteria for inclusion are unclear, political theorist Richard Jackson has argued that "the 'war on terrorism' therefore, is simultaneously a set of actual practices—wars, covert operations, agencies, and institutions—and an accompanying series of assumptions, beliefs, justifications, and narratives—it is an entire language or discourse." Jackson cites among many examples a statement by John Ashcroft that "the attacks of September 11 drew a bright line of demarcation between the civil and the savage". Administration officials also described "terrorists" as hateful, treacherous, barbarous, mad, twisted, perverted, without faith, parasitical, inhuman, and, most commonly, evil. Americans, in contrast, were described as brave, loving, generous, strong, resourceful, heroic, and respectful of human rights. -The origins of al-Qaeda can be traced to the Soviet war in Afghanistan (December 1979 – February 1989). The United States, United Kingdom, Saudi Arabia, Pakistan, and the People's Republic of China supported the Islamist Afghan mujahadeen guerillas against the military forces of the Soviet Union and the Democratic Republic of Afghanistan. A small number of "Afghan Arab" volunteers joined the fight against the Soviets, including Osama bin Laden, but there is no evidence they received any external assistance. In May 1996 the group World Islamic Front for Jihad Against Jews and Crusaders (WIFJAJC), sponsored by bin Laden (and later re-formed as al-Qaeda), started forming a large base of operations in Afghanistan, where the Islamist extremist regime of the Taliban had seized power earlier in the year. In February 1998, Osama bin Laden signed a fatwā, as head of al-Qaeda, declaring war on the West and Israel, later in May of that same year al-Qaeda released a video declaring war on the U.S. and the West. -On 7 August 1998, al-Qaeda struck the U.S. embassies in Kenya and Tanzania, killing 224 people, including 12 Americans. In retaliation, U.S. President Bill Clinton launched Operation Infinite Reach, a bombing campaign in Sudan and Afghanistan against targets the U.S. asserted were associated with WIFJAJC, although others have questioned whether a pharmaceutical plant in Sudan was used as a chemical warfare plant. The plant produced much of the region's antimalarial drugs and around 50% of Sudan's pharmaceutical needs. The strikes failed to kill any leaders of WIFJAJC or the Taliban. -On the morning of 11 September 2001, 19 men affiliated with al-Qaeda hijacked four airliners all bound for California. Once the hijackers assumed control of the airliners, they told the passengers that they had the bomb on board and would spare the lives of passengers and crew once their demands were met – no passenger and crew actually suspected that they would use the airliners as suicide weapons since it had never happened before in history. The hijackers – members of al-Qaeda's Hamburg cell – intentionally crashed two airliners into the Twin Towers of the World Trade Center in New York City. Both buildings collapsed within two hours from fire damage related to the crashes, destroying nearby buildings and damaging others. The hijackers crashed a third airliner into the Pentagon in Arlington County, Virginia, just outside Washington D.C. The fourth plane crashed into a field near Shanksville, Pennsylvania, after some of its passengers and flight crew attempted to retake control of the plane, which the hijackers had redirected toward Washington D.C., to target the White House, or the U.S. Capitol. No flights had survivors. A total of 2,977 victims and the 19 hijackers perished in the attacks. -The Authorization for Use of Military Force Against Terrorists or "AUMF" was made law on 14 September 2001, to authorize the use of United States Armed Forces against those responsible for the attacks on 11 September 2001. It authorized the President to use all necessary and appropriate force against those nations, organizations, or persons he determines planned, authorized, committed, or aided the terrorist attacks that occurred on 11 September 2001, or harbored such organizations or persons, in order to prevent any future acts of international terrorism against the United States by such nations, organizations or persons. Congress declares this is intended to constitute specific statutory authorization within the meaning of section 5(b) of the War Powers Resolution of 1973. -Subsequently, in October 2001, U.S. forces (with UK and coalition allies) invaded Afghanistan to oust the Taliban regime. On 7 October 2001, the official invasion began with British and U.S. forces conducting airstrike campaigns over enemy targets. Kabul, the capital city of Afghanistan, fell by mid-November. The remaining al-Qaeda and Taliban remnants fell back to the rugged mountains of eastern Afghanistan, mainly Tora Bora. In December, Coalition forces (the U.S. and its allies) fought within that region. It is believed that Osama bin Laden escaped into Pakistan during the battle. -The Taliban regrouped in western Pakistan and began to unleash an insurgent-style offensive against Coalition forces in late 2002. Throughout southern and eastern Afghanistan, firefights broke out between the surging Taliban and Coalition forces. Coalition forces responded with a series of military offensives and an increase in the amount of troops in Afghanistan. In February 2010, Coalition forces launched Operation Moshtarak in southern Afghanistan along with other military offensives in the hopes that they would destroy the Taliban insurgency once and for all. Peace talks are also underway between Taliban affiliated fighters and Coalition forces. In September 2014, Afghanistan and the United States signed a security agreement, which permits United States and NATO forces to remain in Afghanistan until at least 2024. The United States and other NATO and non-NATO forces are planning to withdraw; with the Taliban claiming it has defeated the United States and NATO, and the Obama Administration viewing it as a victory. In December 2014, ISAF encasing its colors, and Resolute Support began as the NATO operation in Afghanistan. Continued United States operations within Afghanistan will continue under the name "Operation Freedom's Sentinel". -In January 2002, the United States Special Operations Command, Pacific deployed to the Philippines to advise and assist the Armed Forces of the Philippines in combating Filipino Islamist groups. The operations were mainly focused on removing the Abu Sayyaf group and Jemaah Islamiyah (JI) from their stronghold on the island of Basilan. The second portion of the operation was conducted as a humanitarian program called "Operation Smiles". The goal of the program was to provide medical care and services to the region of Basilan as part of a "Hearts and Minds" program. Joint Special Operations Task Force – Philippines disbanded in June 2014, ending a 14-year mission. After JSOTF-P disbanded, as late as November 2014, American forces continued to operate in the Philippines under the name "PACOM Augmentation Team". -On 14 September 2009, U.S. Special Forces killed two men and wounded and captured two others near the Somali village of Baarawe. Witnesses claim that helicopters used for the operation launched from French-flagged warships, but that could not be confirmed. A Somali-based al-Qaida affiliated group, the Al-Shabaab, has confirmed the death of "sheik commander" Saleh Ali Saleh Nabhan along with an unspecified number of militants. Nabhan, a Kenyan, was wanted in connection with the 2002 Mombasa attacks. -The conflict in northern Mali began in January 2012 with radical Islamists (affiliated to al-Qaeda) advancing into northern Mali. The Malian government had a hard time maintaining full control over their country. The fledgling government requested support from the international community on combating the Islamic militants. In January 2013, France intervened on behalf of the Malian government's request and deployed troops into the region. They launched Operation Serval on 11 January 2013, with the hopes of dislodging the al-Qaeda affiliated groups from northern Mali. -Following the ceasefire agreement that suspended hostilities (but not officially ended) in the 1991 Gulf War, the United States and its allies instituted and began patrolling Iraqi no-fly zones, to protect Iraq's Kurdish and Shi'a Arab population—both of which suffered attacks from the Hussein regime before and after the Gulf War—in Iraq's northern and southern regions, respectively. U.S. forces continued in combat zone deployments through November 1995 and launched Operation Desert Fox against Iraq in 1998 after it failed to meet U.S. demands of "unconditional cooperation" in weapons inspections. -The first ground attack came at the Battle of Umm Qasr on 21 March 2003 when a combined force of British, American and Polish forces seized control of the port city of Umm Qasr. Baghdad, Iraq's capital city, fell to American forces in April 2003 and Saddam Hussein's government quickly dissolved. On 1 May 2003, Bush announced that major combat operations in Iraq had ended. However, an insurgency arose against the U.S.-led coalition and the newly developing Iraqi military and post-Saddam government. The insurgency, which included al-Qaeda affiliated groups, led to far more coalition casualties than the invasion. Other elements of the insurgency were led by fugitive members of President Hussein's Ba'ath regime, which included Iraqi nationalists and pan-Arabists. Many insurgency leaders are Islamists and claim to be fighting a religious war to reestablish the Islamic Caliphate of centuries past. Iraq's former president, Saddam Hussein was captured by U.S. forces in December 2003. He was executed in 2006. -In a major split in the ranks of Al Qaeda's organization, the Iraqi franchise, known as Al Qaeda in Iraq covertly invaded Syria and the Levant and began participating in the ongoing Syrian Civil War, gaining enough support and strength to re-invade Iraq's western provinces under the name of the Islamic State of Iraq and the Levant (ISIS/ISIL), taking over much of the country in a blitzkrieg-like action and combining the Iraq insurgency and Syrian Civil War into a single conflict. Due to their extreme brutality and a complete change in their overall ideology, Al Qaeda's core organization in Central Asia eventually denounced ISIS and directed their affiliates to cut off all ties with this organization. Many analysts[who?] believe that because of this schism, Al Qaeda and ISIL are now in a competition to retain the title of the world's most powerful terrorist organization. -The Obama administration began to reengage in Iraq with a series of airstrikes aimed at ISIS beginning on 10 August 2014. On 9 September 2014 President Obama said that he had the authority he needed to take action to destroy the militant group known as the Islamic State of Iraq and the Levant, citing the 2001 Authorization for Use of Military Force Against Terrorists, and thus did not require additional approval from Congress. The following day on 10 September 2014 President Barack Obama made a televised speech about ISIL, which he stated "Our objective is clear: We will degrade, and ultimately destroy, ISIL through a comprehensive and sustained counter-terrorism strategy". Obama has authorized the deployment of additional U.S. Forces into Iraq, as well as authorizing direct military operations against ISIL within Syria. On the night of 21/22 September the United States, Saudi Arabia, Bahrain, the UAE, Jordan and Qatar started air attacks against ISIS in Syria.[citation needed] -Following the 11 September 2001 attacks, former President of Pakistan Pervez Musharraf sided with the U.S. against the Taliban government in Afghanistan after an ultimatum by then U.S. President George W. Bush. Musharraf agreed to give the U.S. the use of three airbases for Operation Enduring Freedom. United States Secretary of State Colin Powell and other U.S. administration officials met with Musharraf. On 19 September 2001, Musharraf addressed the people of Pakistan and stated that, while he opposed military tactics against the Taliban, Pakistan risked being endangered by an alliance of India and the U.S. if it did not cooperate. In 2006, Musharraf testified that this stance was pressured by threats from the U.S., and revealed in his memoirs that he had "war-gamed" the United States as an adversary and decided that it would end in a loss for Pakistan. -On 12 January 2002, Musharraf gave a speech against Islamic extremism. He unequivocally condemned all acts of terrorism and pledged to combat Islamic extremism and lawlessness within Pakistan itself. He stated that his government was committed to rooting out extremism and made it clear that the banned militant organizations would not be allowed to resurface under any new name. He said, "the recent decision to ban extremist groups promoting militancy was taken in the national interest after thorough consultations. It was not taken under any foreign influence". -In 2002, the Musharraf-led government took a firm stand against the jihadi organizations and groups promoting extremism, and arrested Maulana Masood Azhar, head of the Jaish-e-Mohammed, and Hafiz Muhammad Saeed, chief of the Lashkar-e-Taiba, and took dozens of activists into custody. An official ban was imposed on the groups on 12 January. Later that year, the Saudi born Zayn al-Abidn Muhammed Hasayn Abu Zubaydah was arrested by Pakistani officials during a series of joint U.S.-Pakistan raids. Zubaydah is said to have been a high-ranking al-Qaeda official with the title of operations chief and in charge of running al-Qaeda training camps. Other prominent al-Qaeda members were arrested in the following two years, namely Ramzi bin al-Shibh, who is known to have been a financial backer of al-Qaeda operations, and Khalid Sheikh Mohammed, who at the time of his capture was the third highest-ranking official in al-Qaeda and had been directly in charge of the planning for the 11 September attacks. -The use of drones by the Central Intelligence Agency in Pakistan to carry out operations associated with the Global War on Terror sparks debate over sovereignty and the laws of war. The U.S. Government uses the CIA rather than the U.S. Air Force for strikes in Pakistan in order to avoid breaching sovereignty through military invasion. The United States was criticized by[according to whom?] a report on drone warfare and aerial sovereignty for abusing the term 'Global War on Terror' to carry out military operations through government agencies without formally declaring war. -In a 'Letter to American People' written by Osama bin Laden in 2002, he stated that one of the reasons he was fighting America is because of its support of India on the Kashmir issue. While on a trip to Delhi in 2002, U.S. Secretary of Defense Donald Rumsfeld suggested that Al-Qaeda was active in Kashmir, though he did not have any hard evidence. An investigation in 2002 unearthed evidence that Al-Qaeda and its affiliates were prospering in Pakistan-administered Kashmir with tacit approval of Pakistan's National Intelligence agency Inter-Services Intelligence. A team of Special Air Service and Delta Force was sent into Indian-administered Kashmir in 2002 to hunt for Osama bin Laden after reports that he was being sheltered by the Kashmiri militant group Harkat-ul-Mujahideen. U.S. officials believed that Al-Qaeda was helping organize a campaign of terror in Kashmir in order to provoke conflict between India and Pakistan. Fazlur Rehman Khalil, the leader of the Harkat-ul-Mujahideen, signed al-Qaeda's 1998 declaration of holy war, which called on Muslims to attack all Americans and their allies. Indian sources claimed that In 2006, Al-Qaeda claimed they had established a wing in Kashmir; this worried the Indian government. India also claimed that Al-Qaeda has strong ties with the Kashmir militant groups Lashkar-e-Taiba and Jaish-e-Mohammed in Pakistan. While on a visit to Pakistan in January 2010, U.S. Defense secretary Robert Gates stated that Al-Qaeda was seeking to destabilize the region and planning to provoke a nuclear war between India and Pakistan. -In September 2009, a U.S. Drone strike reportedly killed Ilyas Kashmiri, who was the chief of Harkat-ul-Jihad al-Islami, a Kashmiri militant group associated with Al-Qaeda. Kashmiri was described by Bruce Riedel as a 'prominent' Al-Qaeda member, while others described him as the head of military operations for Al-Qaeda. Waziristan had now become the new battlefield for Kashmiri militants, who were now fighting NATO in support of Al-Qaeda. On 8 July 2012, Al-Badar Mujahideen, a breakaway faction of Kashmir centric terror group Hizbul Mujahideen, on conclusion of their two-day Shuhada Conference called for mobilisation of resources for continuation of jihad in Kashmir. -In the following months, NATO took a wide range of measures to respond to the threat of terrorism. On 22 November 2002, the member states of the Euro-Atlantic Partnership Council (EAPC) decided on a Partnership Action Plan against Terrorism, which explicitly states, "EAPC States are committed to the protection and promotion of fundamental freedoms and human rights, as well as the rule of law, in combating terrorism." NATO started naval operations in the Mediterranean Sea designed to prevent the movement of terrorists or weapons of mass destruction as well as to enhance the security of shipping in general called Operation Active Endeavour. -Support for the U.S. cooled when America made clear its determination to invade Iraq in late 2002. Even so, many of the "coalition of the willing" countries that unconditionally supported the U.S.-led military action have sent troops to Afghanistan, particular neighboring Pakistan, which has disowned its earlier support for the Taliban and contributed tens of thousands of soldiers to the conflict. Pakistan was also engaged in the War in North-West Pakistan (Waziristan War). Supported by U.S. intelligence, Pakistan was attempting to remove the Taliban insurgency and al-Qaeda element from the northern tribal areas. -The British 16th Air Assault Brigade (later reinforced by Royal Marines) formed the core of the force in southern Afghanistan, along with troops and helicopters from Australia, Canada and the Netherlands. The initial force consisted of roughly 3,300 British, 2,000 Canadian, 1,400 from the Netherlands and 240 from Australia, along with special forces from Denmark and Estonia and small contingents from other nations. The monthly supply of cargo containers through Pakistani route to ISAF in Afghanistan is over 4,000 costing around 12 billion in Pakistani Rupees. -In addition to military efforts abroad, in the aftermath of 9/11 the Bush Administration increased domestic efforts to prevent future attacks. Various government bureaucracies that handled security and military functions were reorganized. A new cabinet-level agency called the United States Department of Homeland Security was created in November 2002 to lead and coordinate the largest reorganization of the U.S. federal government since the consolidation of the armed forces into the Department of Defense.[citation needed] -The USA PATRIOT Act of October 2001 dramatically reduces restrictions on law enforcement agencies' ability to search telephone, e-mail communications, medical, financial, and other records; eases restrictions on foreign intelligence gathering within the United States; expands the Secretary of the Treasury's authority to regulate financial transactions, particularly those involving foreign individuals and entities; and broadens the discretion of law enforcement and immigration authorities in detaining and deporting immigrants suspected of terrorism-related acts. The act also expanded the definition of terrorism to include domestic terrorism, thus enlarging the number of activities to which the USA PATRIOT Act's expanded law enforcement powers could be applied. A new Terrorist Finance Tracking Program monitored the movements of terrorists' financial resources (discontinued after being revealed by The New York Times). Global telecommunication usage, including those with no links to terrorism, is being collected and monitored through the NSA electronic surveillance program. The Patriot Act is still in effect. -Political interest groups have stated that these laws remove important restrictions on governmental authority, and are a dangerous encroachment on civil liberties, possible unconstitutional violations of the Fourth Amendment. On 30 July 2003, the American Civil Liberties Union (ACLU) filed the first legal challenge against Section 215 of the Patriot Act, claiming that it allows the FBI to violate a citizen's First Amendment rights, Fourth Amendment rights, and right to due process, by granting the government the right to search a person's business, bookstore, and library records in a terrorist investigation, without disclosing to the individual that records were being searched. Also, governing bodies in a number of communities have passed symbolic resolutions against the act. -In 2005, the UN Security Council adopted Resolution 1624 concerning incitement to commit acts of terrorism and the obligations of countries to comply with international human rights laws. Although both resolutions require mandatory annual reports on counter-terrorism activities by adopting nations, the United States and Israel have both declined to submit reports. In the same year, the United States Department of Defense and the Chairman of the Joint Chiefs of Staff issued a planning document, by the name "National Military Strategic Plan for the War on Terrorism", which stated that it constituted the "comprehensive military plan to prosecute the Global War on Terror for the Armed Forces of the United States...including the findings and recommendations of the 9/11 Commission and a rigorous examination with the Department of Defense". -Criticism of the War on Terror addresses the issues, morality, efficiency, economics, and other questions surrounding the War on Terror and made against the phrase itself, calling it a misnomer. The notion of a "war" against "terrorism" has proven highly contentious, with critics charging that it has been exploited by participating governments to pursue long-standing policy/military objectives, reduce civil liberties, and infringe upon human rights. It is argued that the term war is not appropriate in this context (as in War on Drugs), since there is no identifiable enemy, and that it is unlikely international terrorism can be brought to an end by military means. -Other critics, such as Francis Fukuyama, note that "terrorism" is not an enemy, but a tactic; calling it a "war on terror", obscures differences between conflicts such as anti-occupation insurgents and international mujahideen. With a military presence in Iraq and Afghanistan and its associated collateral damage Shirley Williams maintains this increases resentment and terrorist threats against the West. There is also perceived U.S. hypocrisy, media-induced hysteria, and that differences in foreign and security policy have damaged America's image in most of the world. From cd9f15febbcaa39dc6f1b131740934d463a57af9 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Thu, 25 Apr 2024 17:04:16 +0200 Subject: [PATCH 07/13] fixing tests --- e2e/pipelines/test_evaluation_pipeline.py | 15 ++++----- e2e/samples/test_documents/Annelid.txt | 35 -------------------- e2e/samples/test_documents/Strasbourg.txt | 34 +++++++++++++++++++ e2e/samples/test_documents/War_on_Terror.txt | 31 +++++++++++++++++ 4 files changed, 72 insertions(+), 43 deletions(-) delete mode 100644 e2e/samples/test_documents/Annelid.txt create mode 100644 e2e/samples/test_documents/Strasbourg.txt create mode 100644 e2e/samples/test_documents/War_on_Terror.txt diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py index 883b45c88f..3cff1a9bcb 100644 --- a/e2e/pipelines/test_evaluation_pipeline.py +++ b/e2e/pipelines/test_evaluation_pipeline.py @@ -108,7 +108,7 @@ def run_rag_pipeline(documents, evaluation_questions, rag_pipeline_a): "answer_builder": {"query": q["question"]}, } ) - truth_docs.append([doc for doc in documents if doc.meta["name"] in q["ground_truth_doc"]]) + truth_docs.append([doc for doc in documents if doc.meta["name"] in q["ground_truth_doc"] and doc.content]) retrieved_docs.append(response["answer_builder"]["answers"][0].documents) contexts.append([doc.content for doc in response["answer_builder"]["answers"][0].documents]) pred_answers.append(response["answer_builder"]["answers"][0].data) @@ -136,7 +136,6 @@ def test_evaluation_pipeline(samples_path): "answer": "Martin Bucer", "ground_truth_doc": ["Strasbourg.txt"], }, - {"question": "What separates many annelids' segments?", "answer": "Septa", "ground_truth_doc": ["Annelid.txt"]}, { "question": "What is materialism?", "answer": "a form of philosophical monism", @@ -149,13 +148,14 @@ def test_evaluation_pipeline(samples_path): for article in os.listdir(full_path): with open(f"{full_path}/{article}", "r") as f: for text in f.read().split("\n"): - docs.append(Document(content=text, meta={"name": article})) + if text: + docs.append(Document(content=text, meta={"name": article})) doc_store = indexing_pipeline(docs) questions = [q["question"] for q in eval_questions] truth_answers = [q["answer"] for q in eval_questions] - rag_pipeline_a = rag_pipeline(doc_store, top_k=3) + rag_pipeline_a = rag_pipeline(doc_store, top_k=2) contexts_a, pred_answers_a, retrieved_docs_a, truth_docs = run_rag_pipeline(docs, eval_questions, rag_pipeline_a) results_rag_a = evaluation_pipeline( questions, truth_docs, truth_answers, retrieved_docs_a, contexts_a, pred_answers_a @@ -219,9 +219,9 @@ def test_evaluation_pipeline(samples_path): "Document Recall Single Hit", "Document Recall Multi Hit", ] - assert len(df) == 4 + assert len(df) == 3 - rag_pipeline_b = rag_pipeline(doc_store, top_k=5) + rag_pipeline_b = rag_pipeline(doc_store, top_k=4) contexts_b, pred_answers_b, retrieved_docs_b, truth_docs = run_rag_pipeline(docs, eval_questions, rag_pipeline_b) results_rag_b = evaluation_pipeline( questions, truth_docs, truth_answers, retrieved_docs_b, contexts_b, pred_answers_b @@ -261,5 +261,4 @@ def test_evaluation_pipeline(samples_path): } evaluation_result_b = EvaluationRunResult(run_name="rag_pipeline_b", results=results_b, inputs=inputs_b) df_comparative = evaluation_result_a.comparative_individual_scores_report(evaluation_result_b) - - print(df_comparative) + assert len(df_comparative) == 3 diff --git a/e2e/samples/test_documents/Annelid.txt b/e2e/samples/test_documents/Annelid.txt deleted file mode 100644 index 8cf40a1bb4..0000000000 --- a/e2e/samples/test_documents/Annelid.txt +++ /dev/null @@ -1,35 +0,0 @@ -The annelids are bilaterally symmetrical, triploblastic, coelomate, invertebrate organisms. They also have parapodia for locomotion. Most textbooks still use the traditional division into polychaetes (almost all marine), oligochaetes (which include earthworms) and leech-like species. Cladistic research since 1997 has radically changed this scheme, viewing leeches as a sub-group of oligochaetes and oligochaetes as a sub-group of polychaetes. In addition, the Pogonophora, Echiura and Sipuncula, previously regarded as separate phyla, are now regarded as sub-groups of polychaetes. Annelids are considered members of the Lophotrochozoa, a "super-phylum" of protostomes that also includes molluscs, brachiopods, flatworms and nemerteans. -The basic annelid form consists of multiple segments. Each segment has the same sets of organs and, in most polychaetes, has a pair of parapodia that many species use for locomotion. Septa separate the segments of many species, but are poorly defined or absent in others, and Echiura and Sipuncula show no obvious signs of segmentation. In species with well-developed septa, the blood circulates entirely within blood vessels, and the vessels in segments near the front ends of these species are often built up with muscles that act as hearts. The septa of such species also enable them to change the shapes of individual segments, which facilitates movement by peristalsis ("ripples" that pass along the body) or by undulations that improve the effectiveness of the parapodia. In species with incomplete septa or none, the blood circulates through the main body cavity without any kind of pump, and there is a wide range of locomotory techniques – some burrowing species turn their pharynges inside out to drag themselves through the sediment. -Although many species can reproduce asexually and use similar mechanisms to regenerate after severe injuries, sexual reproduction is the normal method in species whose reproduction has been studied. The minority of living polychaetes whose reproduction and lifecycles are known produce trochophore larvae, that live as plankton and then sink and metamorphose into miniature adults. Oligochaetes are full hermaphrodites and produce a ring-like cocoon around their bodies, in which the eggs and hatchlings are nourished until they are ready to emerge. -Earthworms are Oligochaetes that support terrestrial food chains both as prey and in some regions are important in aeration and enriching of soil. The burrowing of marine polychaetes, which may constitute up to a third of all species in near-shore environments, encourages the development of ecosystems by enabling water and oxygen to penetrate the sea floor. In addition to improving soil fertility, annelids serve humans as food and as bait. Scientists observe annelids to monitor the quality of marine and fresh water. Although blood-letting is no longer in favor with doctors, some leech species are regarded as endangered species because they have been over-harvested for this purpose in the last few centuries. Ragworms' jaws are now being studied by engineers as they offer an exceptional combination of lightness and strength. -Since annelids are soft-bodied, their fossils are rare – mostly jaws and the mineralized tubes that some of the species secreted. Although some late Ediacaran fossils may represent annelids, the oldest known fossil that is identified with confidence comes from about 518 million years ago in the early Cambrian period. Fossils of most modern mobile polychaete groups appeared by the end of the Carboniferous, about 299 million years ago. Palaeontologists disagree about whether some body fossils from the mid Ordovician, about 472 to 461 million years ago, are the remains of oligochaetes, and the earliest indisputable fossils of the group appear in the Tertiary period, which began 65 million years ago. -No single feature distinguishes Annelids from other invertebrate phyla, but they have a distinctive combination of features. Their bodies are long, with segments that are divided externally by shallow ring-like constrictions called annuli and internally by septa ("partitions") at the same points, although in some species the septa are incomplete and in a few cases missing. Most of the segments contain the same sets of organs, although sharing a common gut, circulatory system and nervous system makes them inter-dependent. Their bodies are covered by a cuticle (outer covering) that does not contain cells but is secreted by cells in the skin underneath, is made of tough but flexible collagen and does not molt – on the other hand arthropods' cuticles are made of the more rigid α-chitin, and molt until the arthropods reach their full size. Most annelids have closed circulatory systems, where the blood makes its entire circuit via blood vessels. -Most of an annelid's body consists of segments that are practically identical, having the same sets of internal organs and external chaetae (Greek χαιτη, meaning "hair") and, in some species, appendages. However, the frontmost and rearmost sections are not regarded as true segments as they do not contain the standard sets of organs and do not develop in the same way as the true segments. The frontmost section, called the prostomium (Greek προ- meaning "in front of" and στομα meaning "mouth") contains the brain and sense organs, while the rearmost, called the pygidium (Greek πυγιδιον, meaning "little tail") or periproct contains the anus, generally on the underside. The first section behind the prostomium, called the peristomium (Greek περι- meaning "around" and στομα meaning "mouth"), is regarded by some zoologists as not a true segment, but in some polychaetes the peristomium has chetae and appendages like those of other segments. -Annelids' cuticles are made of collagen fibers, usually in layers that spiral in alternating directions so that the fibers cross each other. These are secreted by the one-cell deep epidermis (outermost skin layer). A few marine annelids that live in tubes lack cuticles, but their tubes have a similar structure, and mucus-secreting glands in the epidermis protect their skins. Under the epidermis is the dermis, which is made of connective tissue, in other words a combination of cells and non-cellular materials such as collagen. Below this are two layers of muscles, which develop from the lining of the coelom (body cavity): circular muscles make a segment longer and slimmer when they contract, while under them are longitudinal muscles, usually four distinct strips, whose contractions make the segment shorter and fatter. Some annelids also have oblique internal muscles that connect the underside of the body to each side. -The setae ("hairs") of annelids project out from the epidermis to provide traction and other capabilities. The simplest are unjointed and form paired bundles near the top and bottom of each side of each segment. The parapodia ("limbs") of annelids that have them often bear more complex chetae at their tips – for example jointed, comb-like or hooked. Chetae are made of moderately flexible β-chitin and are formed by follicles, each of which has a chetoblast ("hair-forming") cell at the bottom and muscles that can extend or retract the cheta. The chetoblasts produce chetae by forming microvilli, fine hair-like extensions that increase the area available for secreting the cheta. When the cheta is complete, the microvilli withdraw into the chetoblast, leaving parallel tunnels that run almost the full length of the cheta. Hence annelids' chetae are structurally different from the setae ("bristles") of arthropods, which are made of the more rigid α-chitin, have a single internal cavity, and are mounted on flexible joints in shallow pits in the cuticle. -Nearly all polychaetes have parapodia that function as limbs, while other major annelid groups lack them. Parapodia are unjointed paired extensions of the body wall, and their muscles are derived from the circular muscles of the body. They are often supported internally by one or more large, thick chetae. The parapodia of burrowing and tube-dwelling polychaetes are often just ridges whose tips bear hooked chetae. In active crawlers and swimmers the parapodia are often divided into large upper and lower paddles on a very short trunk, and the paddles are generally fringed with chetae and sometimes with cirri (fused bundles of cilia) and gills. -The brain generally forms a ring round the pharynx (throat), consisting of a pair of ganglia (local control centers) above and in front of the pharynx, linked by nerve cords either side of the pharynx to another pair of ganglia just below and behind it. The brains of polychaetes are generally in the prostomium, while those of clitellates are in the peristomium or sometimes the first segment behind the peristomium. In some very mobile and active polychaetes the brain is enlarged and more complex, with visible hindbrain, midbrain and forebrain sections. The rest of the central nervous system is generally "ladder-like", consisting of a pair of nerve cords that run through the bottom part of the body and have in each segment paired ganglia linked by a transverse connection. From each segmental ganglion a branching system of local nerves runs into the body wall and then encircles the body. However, in most polychaetes the two main nerve cords are fused, and in the tube-dwelling genus Owenia the single nerve chord has no ganglia and is located in the epidermis. -As in arthropods, each muscle fiber (cell) is controlled by more than one neuron, and the speed and power of the fiber's contractions depends on the combined effects of all its neurons. Vertebrates have a different system, in which one neuron controls a group of muscle fibers. Most annelids' longitudinal nerve trunks include giant axons (the output signal lines of nerve cells). Their large diameter decreases their resistance, which allows them to transmit signals exceptionally fast. This enables these worms to withdraw rapidly from danger by shortening their bodies. Experiments have shown that cutting the giant axons prevents this escape response but does not affect normal movement. -The sensors are primarily single cells that detect light, chemicals, pressure waves and contact, and are present on the head, appendages (if any) and other parts of the body. Nuchal ("on the neck") organs are paired, ciliated structures found only in polychaetes, and are thought to be chemosensors. Some polychaetes also have various combinations of ocelli ("little eyes") that detect the direction from which light is coming and camera eyes or compound eyes that can probably form images. The compound eyes probably evolved independently of arthropods' eyes. Some tube-worms use ocelli widely spread over their bodies to detect the shadows of fish, so that they can quickly withdraw into their tubes. Some burrowing and tube-dwelling polychaetes have statocysts (tilt and balance sensors) that tell them which way is down. A few polychaete genera have on the undersides of their heads palps that are used both in feeding and as "feelers", and some of these also have antennae that are structurally similar but probably are used mainly as "feelers". -Most annelids have a pair of coelomata (body cavities) in each segment, separated from other segments by septa and from each other by vertical mesenteries. Each septum forms a sandwich with connective tissue in the middle and mesothelium (membrane that serves as a lining) from the preceding and following segments on either side. Each mesentery is similar except that the mesothelium is the lining of each of the pair of coelomata, and the blood vessels and, in polychaetes, the main nerve cords are embedded in it. The mesothelium is made of modified epitheliomuscular cells; in other words, their bodies form part of the epithelium but their bases extend to form muscle fibers in the body wall. The mesothelium may also form radial and circular muscles on the septa, and circular muscles around the blood vessels and gut. Parts of the mesothelium, especially on the outside of the gut, may also form chloragogen cells that perform similar functions to the livers of vertebrates: producing and storing glycogen and fat; producing the oxygen-carrier hemoglobin; breaking down proteins; and turning nitrogenous waste products into ammonia and urea to be excreted. -Many annelids move by peristalsis (waves of contraction and expansion that sweep along the body), or flex the body while using parapodia to crawl or swim. In these animals the septa enable the circular and longitudinal muscles to change the shape of individual segments, by making each segment a separate fluid-filled "balloon". However, the septa are often incomplete in annelids that are semi-sessile or that do not move by peristalsis or by movements of parapodia – for example some move by whipping movements of the body, some small marine species move by means of cilia (fine muscle-powered hairs) and some burrowers turn their pharynges (throats) inside out to penetrate the sea-floor and drag themselves into it. -The fluid in the coelomata contains coelomocyte cells that defend the animals against parasites and infections. In some species coelomocytes may also contain a respiratory pigment – red hemoglobin in some species, green chlorocruorin in others (dissolved in the plasma) – and provide oxygen transport within their segments. Respiratory pigment is also dissolved in the blood plasma. Species with well-developed septa generally also have blood vessels running all long their bodies above and below the gut, the upper one carrying blood forwards while the lower one carries it backwards. Networks of capillaries in the body wall and around the gut transfer blood between the main blood vessels and to parts of the segment that need oxygen and nutrients. Both of the major vessels, especially the upper one, can pump blood by contracting. In some annelids the forward end of the upper blood vessel is enlarged with muscles to form a heart, while in the forward ends of many earthworms some of the vessels that connect the upper and lower main vessels function as hearts. Species with poorly developed or no septa generally have no blood vessels and rely on the circulation within the coelom for delivering nutrients and oxygen. -However, leeches and their closest relatives have a body structure that is very uniform within the group but significantly different from that of other annelids, including other members of the Clitellata. In leeches there are no septa, the connective tissue layer of the body wall is so thick that it occupies much of the body, and the two coelomata are widely separated and run the length of the body. They function as the main blood vessels, although they are side-by-side rather than upper and lower. However, they are lined with mesothelium, like the coelomata and unlike the blood vessels of other annelids. Leeches generally use suckers at their front and rear ends to move like inchworms. The anus is on the upper surface of the pygidium. -Feeding structures in the mouth region vary widely, and have little correlation with the animals' diets. Many polychaetes have a muscular pharynx that can be everted (turned inside out to extend it). In these animals the foremost few segments often lack septa so that, when the muscles in these segments contract, the sharp increase in fluid pressure from all these segments everts the pharynx very quickly. Two families, the Eunicidae and Phyllodocidae, have evolved jaws, which can be used for seizing prey, biting off pieces of vegetation, or grasping dead and decaying matter. On the other hand, some predatory polychaetes have neither jaws nor eversible pharynges. Selective deposit feeders generally live in tubes on the sea-floor and use palps to find food particles in the sediment and then wipe them into their mouths. Filter feeders use "crowns" of palps covered in cilia that wash food particles towards their mouths. Non-selective deposit feeders ingest soil or marine sediments via mouths that are generally unspecialized. Some clitellates have sticky pads in the roofs of their mouths, and some of these can evert the pads to capture prey. Leeches often have an eversible proboscis, or a muscular pharynx with two or three teeth. -The gut is generally an almost straight tube supported by the mesenteries (vertical partitions within segments), and ends with the anus on the underside of the pygidium. However, in members of the tube-dwelling family Siboglinidae the gut is blocked by a swollen lining that houses symbiotic bacteria, which can make up 15% of the worms' total weight. The bacteria convert inorganic matter – such as hydrogen sulfide and carbon dioxide from hydrothermal vents, or methane from seeps – to organic matter that feeds themselves and their hosts, while the worms extend their palps into the gas flows to absorb the gases needed by the bacteria. -Annelids with blood vessels use metanephridia to remove soluble waste products, while those without use protonephridia. Both of these systems use a two-stage filtration process, in which fluid and waste products are first extracted and these are filtered again to re-absorb any re-usable materials while dumping toxic and spent materials as urine. The difference is that protonephridia combine both filtration stages in the same organ, while metanephridia perform only the second filtration and rely on other mechanisms for the first – in annelids special filter cells in the walls of the blood vessels let fluids and other small molecules pass into the coelomic fluid, where it circulates to the metanephridia. In annelids the points at which fluid enters the protonephridia or metanephridia are on the forward side of a septum while the second-stage filter and the nephridiopore (exit opening in the body wall) are in the following segment. As a result, the hindmost segment (before the growth zone and pygidium) has no structure that extracts its wastes, as there is no following segment to filter and discharge them, while the first segment contains an extraction structure that passes wastes to the second, but does not contain the structures that re-filter and discharge urine. -It is thought that annelids were originally animals with two separate sexes, which released ova and sperm into the water via their nephridia. The fertilized eggs develop into trochophore larvae, which live as plankton. Later they sink to the sea-floor and metamorphose into miniature adults: the part of the trochophore between the apical tuft and the prototroch becomes the prostomium (head); a small area round the trochophore's anus becomes the pygidium (tail-piece); a narrow band immediately in front of that becomes the growth zone that produces new segments; and the rest of the trochophore becomes the peristomium (the segment that contains the mouth). -However, the lifecycles of most living polychaetes, which are almost all marine animals, are unknown, and only about 25% of the 300+ species whose lifecycles are known follow this pattern. About 14% use a similar external fertilization but produce yolk-rich eggs, which reduce the time the larva needs to spend among the plankton, or eggs from which miniature adults emerge rather than larvae. The rest care for the fertilized eggs until they hatch – some by producing jelly-covered masses of eggs which they tend, some by attaching the eggs to their bodies and a few species by keeping the eggs within their bodies until they hatch. These species use a variety of methods for sperm transfer; for example, in some the females collect sperm released into the water, while in others the males have a penis that inject sperm into the female. There is no guarantee that this is a representative sample of polychaetes' reproductive patterns, and it simply reflects scientists' current knowledge. -Some polychaetes breed only once in their lives, while others breed almost continuously or through several breeding seasons. While most polychaetes remain of one sex all their lives, a significant percentage of species are full hermaphrodites or change sex during their lives. Most polychaetes whose reproduction has been studied lack permanent gonads, and it is uncertain how they produce ova and sperm. In a few species the rear of the body splits off and becomes a separate individual that lives just long enough to swim to a suitable environment, usually near the surface, and spawn. -Most mature clitellates (the group that includes earthworms and leeches) are full hermaphrodites, although in a few leech species younger adults function as males and become female at maturity. All have well-developed gonads, and all copulate. Earthworms store their partners' sperm in spermathecae ("sperm stores") and then the clitellum produces a cocoon that collects ova from the ovaries and then sperm from the spermathecae. Fertilization and development of earthworm eggs takes place in the cocoon. Leeches' eggs are fertilized in the ovaries, and then transferred to the cocoon. In all clitellates the cocoon also either produces yolk when the eggs are fertilized or nutrients while they are developing. All clitellates hatch as miniature adults rather than larvae. -Charles Darwin's book The Formation of Vegetable Mould through the Action of Worms (1881) presented the first scientific analysis of earthworms' contributions to soil fertility. Some burrow while others live entirely on the surface, generally in moist leaf litter. The burrowers loosen the soil so that oxygen and water can penetrate it, and both surface and burrowing worms help to produce soil by mixing organic and mineral matter, by accelerating the decomposition of organic matter and thus making it more quickly available to other organisms, and by concentrating minerals and converting them to forms that plants can use more easily. Earthworms are also important prey for birds ranging in size from robins to storks, and for mammals ranging from shrews to badgers, and in some cases conserving earthworms may be essential for conserving endangered birds. -Terrestrial annelids can be invasive in some situations. In the glaciated areas of North America, for example, almost all native earthworms are thought to have been killed by the glaciers and the worms currently found in those areas are all introduced from other areas, primarily from Europe, and, more recently, from Asia. Northern hardwood forests are especially negatively impacted by invasive worms through the loss of leaf duff, soil fertility, changes in soil chemistry and the loss of ecological diversity. Especially of concern is Amynthas agrestis and at least one state (Wisconsin) has listed it as a prohibited species. -Earthworms make a significant contribution to soil fertility. The rear end of the Palolo worm, a marine polychaete that tunnels through coral, detaches in order to spawn at the surface, and the people of Samoa regard these spawning modules as a delicacy. Anglers sometimes find that worms are more effective bait than artificial flies, and worms can be kept for several days in a tin lined with damp moss. Ragworms are commercially important as bait and as food sources for aquaculture, and there have been proposals to farm them in order to reduce over-fishing of their natural populations. Some marine polychaetes' predation on molluscs causes serious losses to fishery and aquaculture operations. -Accounts of the use of leeches for the medically dubious practise of blood-letting have come from China around 30 AD, India around 200 AD, ancient Rome around 50 AD and later throughout Europe. In the 19th century medical demand for leeches was so high that some areas' stocks were exhausted and other regions imposed restrictions or bans on exports, and Hirudo medicinalis is treated as an endangered species by both IUCN and CITES. More recently leeches have been used to assist in microsurgery, and their saliva has provided anti-inflammatory compounds and several important anticoagulants, one of which also prevents tumors from spreading. -Since annelids are soft-bodied, their fossils are rare. Polychaetes' fossil record consists mainly of the jaws that some species had and the mineralized tubes that some secreted. Some Ediacaran fossils such as Dickinsonia in some ways resemble polychaetes, but the similarities are too vague for these fossils to be classified with confidence. The small shelly fossil Cloudina, from 549 to 542 million years ago, has been classified by some authors as an annelid, but by others as a cnidarian (i.e. in the phylum to which jellyfish and sea anemones belong). Until 2008 the earliest fossils widely accepted as annelids were the polychaetes Canadia and Burgessochaeta, both from Canada's Burgess Shale, formed about 505 million years ago in the early Cambrian. Myoscolex, found in Australia and a little older than the Burgess Shale, was possibly an annelid. However, it lacks some typical annelid features and has features which are not usually found in annelids and some of which are associated with other phyla. Then Simon Conway Morris and John Peel reported Phragmochaeta from Sirius Passet, about 518 million years old, and concluded that it was the oldest annelid known to date. There has been vigorous debate about whether the Burgess Shale fossil Wiwaxia was a mollusc or an annelid. Polychaetes diversified in the early Ordovician, about 488 to 474 million years ago. It is not until the early Ordovician that the first annelid jaws are found, thus the crown-group cannot have appeared before this date and probably appeared somewhat later. By the end of the Carboniferous, about 299 million years ago, fossils of most of the modern mobile polychaete groups had appeared. Many fossil tubes look like those made by modern sessile polychaetes , but the first tubes clearly produced by polychaetes date from the Jurassic, less than 199 million years ago. -The earliest good evidence for oligochaetes occurs in the Tertiary period, which began 65 million years ago, and it has been suggested that these animals evolved around the same time as flowering plants in the early Cretaceous, from 130 to 90 million years ago. A trace fossil consisting of a convoluted burrow partly filled with small fecal pellets may be evidence that earthworms were present in the early Triassic period from 251 to 245 million years ago. Body fossils going back to the mid Ordovician, from 472 to 461 million years ago, have been tentatively classified as oligochaetes, but these identifications are uncertain and some have been disputed. -Traditionally the annelids have been divided into two major groups, the polychaetes and clitellates. In turn the clitellates were divided into oligochaetes, which include earthworms, and hirudinomorphs, whose best-known members are leeches. For many years there was no clear arrangement of the approximately 80 polychaete families into higher-level groups. In 1997 Greg Rouse and Kristian Fauchald attempted a "first heuristic step in terms of bringing polychaete systematics to an acceptable level of rigour", based on anatomical structures, and divided polychaetes into: -In 2007 Torsten Struck and colleagues compared 3 genes in 81 taxa, of which 9 were outgroups, in other words not considered closely related to annelids but included to give an indication of where the organisms under study are placed on the larger tree of life. For a cross-check the study used an analysis of 11 genes (including the original 3) in 10 taxa. This analysis agreed that clitellates, pogonophorans and echiurans were on various branches of the polychaete family tree. It also concluded that the classification of polychaetes into Scolecida, Canalipalpata and Aciculata was useless, as the members of these alleged groups were scattered all over the family tree derived from comparing the 81 taxa. In addition, it also placed sipunculans, generally regarded at the time as a separate phylum, on another branch of the polychaete tree, and concluded that leeches were a sub-group of oligochaetes rather than their sister-group among the clitellates. Rouse accepted the analyses based on molecular phylogenetics, and their main conclusions are now the scientific consensus, although the details of the annelid family tree remain uncertain. -In addition to re-writing the classification of annelids and 3 previously independent phyla, the molecular phylogenetics analyses undermine the emphasis that decades of previous writings placed on the importance of segmentation in the classification of invertebrates. Polychaetes, which these analyses found to be the parent group, have completely segmented bodies, while polychaetes' echiurans and sipunculan offshoots are not segmented and pogonophores are segmented only in the rear parts of their bodies. It now seems that segmentation can appear and disappear much more easily in the course of evolution than was previously thought. The 2007 study also noted that the ladder-like nervous system, which is associated with segmentation, is less universal previously thought in both annelids and arthropods.[n 2] -Annelids are members of the protostomes, one of the two major superphyla of bilaterian animals – the other is the deuterostomes, which includes vertebrates. Within the protostomes, annelids used to be grouped with arthropods under the super-group Articulata ("jointed animals"), as segmentation is obvious in most members of both phyla. However, the genes that drive segmentation in arthropods do not appear to do the same in annelids. Arthropods and annelids both have close relatives that are unsegmented. It is at least as easy to assume that they evolved segmented bodies independently as it is to assume that the ancestral protostome or bilaterian was segmented and that segmentation disappeared in many descendant phyla. The current view is that annelids are grouped with molluscs, brachiopods and several other phyla that have lophophores (fan-like feeding structures) and/or trochophore larvae as members of Lophotrochozoa. Bryzoa may be the most basal phylum (the one that first became distinctive) within the Lophotrochozoa, and the relationships between the other members are not yet known. Arthropods are now regarded as members of the Ecdysozoa ("animals that molt"), along with some phyla that are unsegmented. -The "Lophotrochozoa" hypothesis is also supported by the fact that many phyla within this group, including annelids, molluscs, nemerteans and flatworms, follow a similar pattern in the fertilized egg's development. When their cells divide after the 4-cell stage, descendants of these 4 cells form a spiral pattern. In these phyla the "fates" of the embryo's cells, in other words the roles their descendants will play in the adult animal, are the same and can be predicted from a very early stage. Hence this development pattern is often described as "spiral determinate cleavage". diff --git a/e2e/samples/test_documents/Strasbourg.txt b/e2e/samples/test_documents/Strasbourg.txt new file mode 100644 index 0000000000..bf64384234 --- /dev/null +++ b/e2e/samples/test_documents/Strasbourg.txt @@ -0,0 +1,34 @@ +Strasbourg (/ˈstræzbɜːrɡ/, French pronunciation: ​[stʁaz.buʁ, stʁas.buʁ]; Alsatian: Strossburi; German: Straßburg, [ˈʃtʁaːsbʊɐ̯k]) is the capital and largest city of the Alsace-Champagne-Ardenne-Lorraine (ACAL) region in eastern France and is the official seat of the European Parliament. Located close to the border with Germany, it is the capital of the Bas-Rhin département. The city and the region of Alsace were historically predominantly Alemannic-speaking, hence the city's Germanic name. In 2013, the city proper had 275,718 inhabitants, Eurométropole de Strasbourg (Greater Strasbourg) had 475,934 inhabitants and the Arrondissement of Strasbourg had 482,384 inhabitants. With a population of 768,868 in 2012, Strasbourg's metropolitan area (only the part of the metropolitan area on French territory) is the ninth largest in France and home to 13% of the ACAL region's inhabitants. The transnational Eurodistrict Strasbourg-Ortenau had a population of 915,000 inhabitants in 2014. +Strasbourg's historic city centre, the Grande Île (Grand Island), was classified a World Heritage site by UNESCO in 1988, the first time such an honour was placed on an entire city centre. Strasbourg is immersed in the Franco-German culture and although violently disputed throughout history, has been a bridge of unity between France and Germany for centuries, especially through the University of Strasbourg, currently the second largest in France, and the coexistence of Catholic and Protestant culture. The largest Islamic place of worship in France, the Strasbourg Grand Mosque, was inaugurated by French Interior Minister Manuel Valls on 27 September 2012. +Strasbourg is situated on the eastern border of France with Germany. This border is formed by the River Rhine, which also forms the eastern border of the modern city, facing across the river to the German town Kehl. The historic core of Strasbourg however lies on the Grande Île in the River Ill, which here flows parallel to, and roughly 4 kilometres (2.5 mi) from, the Rhine. The natural courses of the two rivers eventually join some distance downstream of Strasbourg, although several artificial waterways now connect them within the city. +The Romans under Nero Claudius Drusus established a military outpost belonging to the Germania Superior Roman province at Strasbourg's current location, and named it Argentoratum. (Hence the town is commonly called Argentina in medieval Latin.) The name "Argentoratum" was first mentioned in 12 BC and the city celebrated its 2,000th birthday in 1988. "Argentorate" as the toponym of the Gaulish settlement preceded it before being Latinized, but it is not known by how long. The Roman camp was destroyed by fire and rebuilt six times between the first and the fifth centuries AD: in 70, 97, 235, 355, in the last quarter of the fourth century, and in the early years of the fifth century. It was under Trajan and after the fire of 97 that Argentoratum received its most extended and fortified shape. From the year 90 on, the Legio VIII Augusta was permanently stationed in the Roman camp of Argentoratum. It then included a cavalry section and covered an area of approximately 20 hectares. Other Roman legions temporarily stationed in Argentoratum were the Legio XIV Gemina and the Legio XXI Rapax, the latter during the reign of Nero. +The centre of Argentoratum proper was situated on the Grande Île (Cardo: current Rue du Dôme, Decumanus: current Rue des Hallebardes). The outline of the Roman "castrum" is visible in the street pattern in the Grande Ile. Many Roman artifacts have also been found along the current Route des Romains, the road that led to Argentoratum, in the suburb of Kœnigshoffen. This was where the largest burial places were situated, as well as the densest concentration of civilian dwelling places and commerces next to the camp. Among the most outstanding finds in Kœnigshoffen were (found in 1911–12) the fragments of a grand Mithraeum that had been shattered by early Christians in the fourth century. From the fourth century, Strasbourg was the seat of the Bishopric of Strasbourg (made an Archbishopric in 1988). Archaeological excavations below the current Église Saint-Étienne in 1948 and 1956 unearthed the apse of a church dating back to the late fourth or early fifth century, considered to be the oldest church in Alsace. It is supposed that this was the first seat of the Roman Catholic Diocese of Strasbourg. +In the fifth century Strasbourg was occupied successively by Alemanni, Huns, and Franks. In the ninth century it was commonly known as Strazburg in the local language, as documented in 842 by the Oaths of Strasbourg. This trilingual text contains, alongside texts in Latin and Old High German (teudisca lingua), the oldest written variety of Gallo-Romance (lingua romana) clearly distinct from Latin, the ancestor of Old French. The town was also called Stratisburgum or Strateburgus in Latin, from which later came Strossburi in Alsatian and Straßburg in Standard German, and then Strasbourg in French. The Oaths of Strasbourg is considered as marking the birth of the two countries of France and Germany with the division of the Carolingian Empire. +A revolution in 1332 resulted in a broad-based city government with participation of the guilds, and Strasbourg declared itself a free republic. The deadly bubonic plague of 1348 was followed on 14 February 1349 by one of the first and worst pogroms in pre-modern history: over a thousand Jews were publicly burnt to death, with the remainder of the Jewish population being expelled from the city. Until the end of the 18th century, Jews were forbidden to remain in town after 10 pm. The time to leave the city was signalled by a municipal herald blowing the Grüselhorn (see below, Museums, Musée historique);. A special tax, the Pflastergeld (pavement money), was furthermore to be paid for any horse that a Jew would ride or bring into the city while allowed to. +In the 1520s during the Protestant Reformation, the city, under the political guidance of Jacob Sturm von Sturmeck and the spiritual guidance of Martin Bucer embraced the religious teachings of Martin Luther. Their adherents established a Gymnasium, headed by Johannes Sturm, made into a University in the following century. The city first followed the Tetrapolitan Confession, and then the Augsburg Confession. Protestant iconoclasm caused much destruction to churches and cloisters, notwithstanding that Luther himself opposed such a practice. Strasbourg was a centre of humanist scholarship and early book-printing in the Holy Roman Empire, and its intellectual and political influence contributed much to the establishment of Protestantism as an accepted denomination in the southwest of Germany. (John Calvin spent several years as a political refugee in the city). The Strasbourg Councillor Sturm and guildmaster Matthias represented the city at the Imperial Diet of Speyer (1529), where their protest led to the schism of the Catholic Church and the evolution of Protestantism. Together with four other free cities, Strasbourg presented the confessio tetrapolitana as its Protestant book of faith at the Imperial Diet of Augsburg in 1530, where the slightly different Augsburg Confession was also handed over to Charles V, Holy Roman Emperor. +Louis' advisors believed that, as long as Strasbourg remained independent, it would endanger the King's newly annexed territories in Alsace, and, that to defend these large rural lands effectively, a garrison had to be placed in towns such as Strasbourg. Indeed, the bridge over the Rhine at Strasbourg had been used repeatedly by Imperial (Holy Roman Empire) forces, and three times during the Franco-Dutch War Strasbourg had served as a gateway for Imperial invasions into Alsace. In September 1681 Louis' forces, though lacking a clear casus belli, surrounded the city with overwhelming force. After some negotiation, Louis marched into the city unopposed on 30 September 1681 and proclaimed its annexation. +This annexation was one of the direct causes of the brief and bloody War of the Reunions whose outcome left the French in possession. The French annexation was recognized by the Treaty of Ryswick (1697). The official policy of religious intolerance which drove most Protestants from France after the revocation of the Edict of Nantes in 1685 was not applied in Strasbourg and in Alsace, because both had a special status as a province à l'instar de l'étranger effectif (a kind of foreign province of the king of France). Strasbourg Cathedral, however, was taken from the Lutherans to be returned to the Catholics as the French authorities tried to promote Catholicism wherever they could (some other historic churches remained in Protestant hands). Its language also remained overwhelmingly German: the German Lutheran university persisted until the French Revolution. Famous students included Goethe and Herder. +Strasbourg's status as a free city was revoked by the French Revolution. Enragés, most notoriously Eulogius Schneider, ruled the city with an increasingly iron hand. During this time, many churches and monasteries were either destroyed or severely damaged. The cathedral lost hundreds of its statues (later replaced by copies in the 19th century) and in April 1794, there was talk of tearing its spire down, on the grounds that it was against the principle of equality. The tower was saved, however, when in May of the same year citizens of Strasbourg crowned it with a giant tin Phrygian cap. This artifact was later kept in the historical collections of the city until it was destroyed by the Germans in 1870 during the Franco-Prussian war. +During the Franco-Prussian War and the Siege of Strasbourg, the city was heavily bombarded by the Prussian army. The bombardment of the city was meant to break the morale of the people of Strasbourg. On 24 and 26 August 1870, the Museum of Fine Arts was destroyed by fire, as was the Municipal Library housed in the Gothic former Dominican church, with its unique collection of medieval manuscripts (most famously the Hortus deliciarum), rare Renaissance books, archeological finds and historical artifacts. The gothic cathedral was damaged as well as the medieval church of Temple Neuf, the theatre, the city hall, the court of justice and many houses. At the end of the siege 10,000 inhabitants were left without shelter; over 600 died, including 261 civilians, and 3200 were injured, including 1,100 civilians. +In 1871, after the end of the war, the city was annexed to the newly established German Empire as part of the Reichsland Elsass-Lothringen under the terms of the Treaty of Frankfurt. As part of Imperial Germany, Strasbourg was rebuilt and developed on a grand and representative scale, such as the Neue Stadt, or "new city" around the present Place de la République. Historian Rodolphe Reuss and Art historian Wilhelm von Bode were in charge of rebuilding the municipal archives, libraries and museums. The University, founded in 1567 and suppressed during the French Revolution as a stronghold of German sentiment,[citation needed] was reopened in 1872 under the name Kaiser-Wilhelms-Universität. +A belt of massive fortifications was established around the city, most of which still stands today, renamed after French generals and generally classified as Monuments historiques; most notably Fort Roon (now Fort Desaix) and Fort Podbielski (now Fort Ducrot) in Mundolsheim, Fort von Moltke (now Fort Rapp) in Reichstett, Fort Bismarck (now Fort Kléber) in Wolfisheim, Fort Kronprinz (now Fort Foch) in Niederhausbergen, Fort Kronprinz von Sachsen (now Fort Joffre) in Holtzheim and Fort Großherzog von Baden (now Fort Frère) in Oberhausbergen. +Following the defeat of the German empire in World War I and the abdication of the German Emperor, some revolutionary insurgents declared Alsace-Lorraine as an independent Republic, without preliminary referendum or vote. On 11 November 1918 (Armistice Day), communist insurgents proclaimed a "soviet government" in Strasbourg, following the example of Kurt Eisner in Munich as well as other German towns. French troops commanded by French general Henri Gouraud entered triumphantly in the city on 22 November. A major street of the city now bears the name of that date (Rue du 22 Novembre) which celebrates the entry of the French in the city. Viewing the massive cheering crowd gathered under the balcony of Strasbourg's town hall, French President Raymond Poincaré stated that "the plebiscite is done". +In 1919, following the Treaty of Versailles, the city was restituted to France in accordance with U.S. President Woodrow Wilson's "Fourteen Points" without a referendum. The date of the assignment was retroactively established on Armistice Day. It is doubtful whether a referendum in Strasbourg would have ended in France's favour since the political parties striving for an autonomous Alsace or a connection to France accounted only for a small proportion of votes in the last Reichstag as well as in the local elections. The Alsatian autonomists who were pro French had won many votes in the more rural parts of the region and other towns since the annexation of the region by Germany in 1871. The movement started with the first election for the Reichstag; those elected were called "les députés protestataires", and until the fall of Bismarck in 1890, they were the only deputies elected by the Alsatians to the German parliament demanding the return of those territories to France. At the last Reichstag election in Strasbourg and its periphery, the clear winners were the Social Democrats; the city was the administrative capital of the region, was inhabited by many Germans appointed by the central government in Berlin and its flourishing economy attracted many Germans. This could explain the difference between the rural vote and the one in Strasbourg. After the war, many Germans left Strasbourg and went back to Germany; some of them were denounced by the locals or expelled by the newly appointed authorities. The Saverne Affair was vivid in the memory among the Alsatians. +Between the German invasion of Poland on 1 September 1939 and the Anglo-French declaration of War against the German Reich on 3 September 1939, the entire city (a total of 120,000 people) was evacuated, like other border towns as well. Until the arrival of the Wehrmacht troops mid-June 1940, the city was, for ten months, completely empty, with the exception of the garrisoned soldiers. The Jews of Strasbourg had been evacuated to Périgueux and Limoges, the University had been evacuated to Clermont-Ferrand. +After the ceasefire following the Fall of France in June 1940, Alsace was annexed to Germany and a rigorous policy of Germanisation was imposed upon it by the Gauleiter Robert Heinrich Wagner. When, in July 1940, the first evacuees were allowed to return, only residents of Alsatian origin were admitted. The last Jews were deported on 15 July 1940 and the main synagogue, a huge Romanesque revival building that had been a major architectural landmark with its 54-metre-high dome since its completion in 1897, was set ablaze, then razed. +In September 1940 the first Alsatian resistance movement led by Marcel Weinum called La main noire (The black hand) was created. It was composed by a group of 25 young men aged from 14 to 18 years old who led several attacks against the German occupation. The actions culminated with the attack of the Gauleiter Robert Wagner, the highest commander of Alsace directly under the order of Hitler. In March 1942, Marcel Weinum was prosecuted by the Gestapo and sentenced to be beheaded at the age of 18 in April 1942 in Stuttgart, Germany. His last words will be: "If I have to die, I shall die but with a pure heart". From 1943 the city was bombarded by Allied aircraft. While the First World War had not notably damaged the city, Anglo-American bombing caused extensive destruction in raids of which at least one was allegedly carried out by mistake. In August 1944, several buildings in the Old Town were damaged by bombs, particularly the Palais Rohan, the Old Customs House (Ancienne Douane) and the Cathedral. On 23 November 1944, the city was officially liberated by the 2nd French Armoured Division under General Leclerc. He achieved the oath that he made with his soldiers, after the decisive Capture of Kufra. With the Oath of Kuffra, they swore to keep up the fight until the French flag flew over the Cathedral of Strasbourg. +Many people from Strasbourg were incorporated in the German Army against their will, and were sent to the eastern front, those young men and women were called Malgré-nous. Many tried to escape from the incorporation, join the French Resistance, or desert the Wehrmacht but many couldn't because they were running the risk of having their families sent to work or concentration camps by the Germans. Many of these men, especially those who did not answer the call immediately, were pressured to "volunteer" for service with the SS, often by direct threats on their families. This threat obliged the majority of them to remain in the German army. After the war, the few that survived were often accused of being traitors or collaborationists, because this tough situation was not known in the rest of France, and they had to face the incomprehension of many. In July 1944, 1500 malgré-nous were released from Soviet captivity and sent to Algiers, where they joined the Free French Forces. Nowadays history recognizes the suffering of those people, and museums, public discussions and memorials have been built to commemorate this terrible period of history of this part of Eastern France (Alsace and Moselle). Liberation of Strasbourg took place on 23 November 1944. +In 1949, the city was chosen to be the seat of the Council of Europe with its European Court of Human Rights and European Pharmacopoeia. Since 1952, the European Parliament has met in Strasbourg, which was formally designated its official 'seat' at the Edinburgh meeting of the European Council of EU heads of state and government in December 1992. (This position was reconfirmed and given treaty status in the 1997 Treaty of Amsterdam). However, only the (four-day) plenary sessions of the Parliament are held in Strasbourg each month, with all other business being conducted in Brussels and Luxembourg. Those sessions take place in the Immeuble Louise Weiss, inaugurated in 1999, which houses the largest parliamentary assembly room in Europe and of any democratic institution in the world. Before that, the EP sessions had to take place in the main Council of Europe building, the Palace of Europe, whose unusual inner architecture had become a familiar sight to European TV audiences. In 1992, Strasbourg became the seat of the Franco-German TV channel and movie-production society Arte. +In addition to the cathedral, Strasbourg houses several other medieval churches that have survived the many wars and destructions that have plagued the city: the Romanesque Église Saint-Étienne, partly destroyed in 1944 by Allied bombing raids, the part Romanesque, part Gothic, very large Église Saint-Thomas with its Silbermann organ on which Wolfgang Amadeus Mozart and Albert Schweitzer played, the Gothic Église protestante Saint-Pierre-le-Jeune with its crypt dating back to the seventh century and its cloister partly from the eleventh century, the Gothic Église Saint-Guillaume with its fine early-Renaissance stained glass and furniture, the Gothic Église Saint-Jean, the part Gothic, part Art Nouveau Église Sainte-Madeleine, etc. The Neo-Gothic church Saint-Pierre-le-Vieux Catholique (there is also an adjacent church Saint-Pierre-le-Vieux Protestant) serves as a shrine for several 15th-century wood worked and painted altars coming from other, now destroyed churches and installed there for public display. Among the numerous secular medieval buildings, the monumental Ancienne Douane (old custom-house) stands out. +The German Renaissance has bequeathed the city some noteworthy buildings (especially the current Chambre de commerce et d'industrie, former town hall, on Place Gutenberg), as did the French Baroque and Classicism with several hôtels particuliers (i.e. palaces), among which the Palais Rohan (1742, now housing three museums) is the most spectacular. Other buildings of its kind are the "Hôtel de Hanau" (1736, now the city hall), the Hôtel de Klinglin (1736, now residence of the préfet), the Hôtel des Deux-Ponts (1755, now residence of the military governor), the Hôtel d'Andlau-Klinglin (1725, now seat of the administration of the Port autonome de Strasbourg) etc. The largest baroque building of Strasbourg though is the 150 m (490 ft) long 1720s main building of the Hôpital civil. As for French Neo-classicism, it is the Opera House on Place Broglie that most prestigiously represents this style. +Strasbourg also offers high-class eclecticist buildings in its very extended German district, the Neustadt, being the main memory of Wilhelmian architecture since most of the major cities in Germany proper suffered intensive damage during World War II. Streets, boulevards and avenues are homogeneous, surprisingly high (up to seven stories) and broad examples of German urban lay-out and of this architectural style that summons and mixes up five centuries of European architecture as well as Neo-Egyptian, Neo-Greek and Neo-Babylonian styles. The former imperial palace Palais du Rhin, the most political and thus heavily criticized of all German Strasbourg buildings epitomizes the grand scale and stylistic sturdiness of this period. But the two most handsome and ornate buildings of these times are the École internationale des Pontonniers (the former Höhere Mädchenschule, girls college) with its towers, turrets and multiple round and square angles and the École des Arts décoratifs with its lavishly ornate façade of painted bricks, woodwork and majolica. +As for modern and contemporary architecture, Strasbourg possesses some fine Art Nouveau buildings (such as the huge Palais des Fêtes and houses and villas like Villa Schutzenberger and Hôtel Brion), good examples of post-World War II functional architecture (the Cité Rotterdam, for which Le Corbusier did not succeed in the architectural contest) and, in the very extended Quartier Européen, some spectacular administrative buildings of sometimes utterly large size, among which the European Court of Human Rights building by Richard Rogers is arguably the finest. Other noticeable contemporary buildings are the new Music school Cité de la Musique et de la Danse, the Musée d'Art moderne et contemporain and the Hôtel du Département facing it, as well as, in the outskirts, the tramway-station Hoenheim-Nord designed by Zaha Hadid. +Strasbourg features a number of prominent parks, of which several are of cultural and historical interest: the Parc de l'Orangerie, laid out as a French garden by André le Nôtre and remodeled as an English garden on behalf of Joséphine de Beauharnais, now displaying noteworthy French gardens, a neo-classical castle and a small zoo; the Parc de la Citadelle, built around impressive remains of the 17th-century fortress erected close to the Rhine by Vauban; the Parc de Pourtalès, laid out in English style around a baroque castle (heavily restored in the 19th century) that now houses a small three-star hotel, and featuring an open-air museum of international contemporary sculpture. The Jardin botanique de l'Université de Strasbourg (botanical garden) was created under the German administration next to the Observatory of Strasbourg, built in 1881, and still owns some greenhouses of those times. The Parc des Contades, although the oldest park of the city, was completely remodeled after World War II. The futuristic Parc des Poteries is an example of European park-conception in the late 1990s. The Jardin des deux Rives, spread over Strasbourg and Kehl on both sides of the Rhine opened in 2004 and is the most extended (60-hectare) park of the agglomeration. The most recent park is Parc du Heyritz (8,7 ha), opened in 2014 along a canal facing the hôpital civil. +Unlike most other cities, Strasbourg's collections of European art are divided into several museums according not only to type and area, but also to epoch. Old master paintings from the Germanic Rhenish territories and until 1681 are displayed in the Musée de l'Œuvre Notre-Dame, old master paintings from all the rest of Europe (including the Dutch Rhenish territories) and until 1871 as well as old master paintings from the Germanic Rhenish territories between 1681 and 1871 are displayed in the Musée des Beaux-Arts. Old master graphic arts until 1871 is displayed in the Cabinet des estampes et dessins. Decorative arts until 1681 ("German period") are displayed in the Musée de l'Œuvre Notre-Dame, decorative arts from 1681 to 1871 ("French period") are displayed in the Musée des Arts décoratifs. International art (painting, sculpture, graphic arts) and decorative art since 1871 is displayed in the Musée d'art moderne et contemporain. The latter museum also displays the city's photographic library. +Strasbourg, well known as centre of humanism, has a long history of excellence in higher-education, at the crossroads of French and German intellectual traditions. Although Strasbourg had been annexed by the Kingdom of France in 1683, it still remained connected to the German-speaking intellectual world throughout the 18th century and the university attracted numerous students from the Holy Roman Empire, including Goethe, Metternich and Montgelas, who studied law in Strasbourg, among the most prominent. Nowadays, Strasbourg is known to offer among the best university courses in France, after Paris. +The Bibliothèque nationale et universitaire (BNU) is, with its collection of more than 3,000,000 titles, the second largest library in France after the Bibliothèque nationale de France. It was founded by the German administration after the complete destruction of the previous municipal library in 1871 and holds the unique status of being simultaneously a students' and a national library. The Strasbourg municipal library had been marked erroneously as "City Hall" in a French commercial map, which had been captured and used by the German artillery to lay their guns. A librarian from Munich later pointed out "...that the destruction of the precious collection was not the fault of a German artillery officer, who used the French map, but of the slovenly and inaccurate scholarship of a Frenchman." +As one of the earliest centers of book-printing in Europe (see above: History), Strasbourg for a long time held a large number of incunabula—documents printed before 1500—in her library as one of her most precious heritages. After the total destruction of this institution in 1870, however, a new collection had to be reassembled from scratch. Today, Strasbourg's different public and institutional libraries again display a sizable total number of incunabula, distributed as follows: Bibliothèque nationale et universitaire, ca. 2 098 Médiathèque de la ville et de la communauté urbaine de Strasbourg, 394 Bibliothèque du Grand Séminaire, 238 Médiathèque protestante, 94 and Bibliothèque alsatique du Crédit Mutuel, 5. +City transportation in Strasbourg includes the futurist-looking Strasbourg tramway that opened in 1994 and is operated by the regional transit company Compagnie des Transports Strasbourgeois (CTS), consisting of 6 lines with a total length of 55.8 km (34.7 mi). The CTS also operates a comprehensive bus network throughout the city that is integrated with the trams. With more than 500 km (311 mi) of bicycle paths, biking in the city is convenient and the CTS operates a cheap bike-sharing scheme named Vélhop'. The CTS, and its predecessors, also operated a previous generation of tram system between 1878 and 1960, complemented by trolleybus routes between 1939 and 1962. +Being a city on the Ill and close to the Rhine, Strasbourg has always been an important centre of fluvial navigation, as is attested by archeological findings. In 1682 the Canal de la Bruche was added to the river navigations, initially to provide transport for sandstone from quarries in the Vosges for use in the fortification of the city. That canal has since closed, but the subsequent Canal du Rhone au Rhine, Canal de la Marne au Rhin and Grand Canal d'Alsace are still in use, as is the important activity of the Port autonome de Strasbourg. Water tourism inside the city proper attracts hundreds of thousands of tourists yearly. +The tram system that now criss-crosses the historic city centre complements walking and biking in it. The centre has been transformed into a pedestrian priority zone that enables and invites walking and biking by making these active modes of transport comfortable, safe and enjoyable. These attributes are accomplished by applying the principle of "filtered permeability" to the existing irregular network of streets. It means that the network adaptations favour active transportation and, selectively, "filter out" the car by reducing the number of streets that run through the centre. While certain streets are discontinuous for cars, they connect to a network of pedestrian and bike paths which permeate the entire centre. In addition, these paths go through public squares and open spaces increasing the enjoyment of the trip. This logic of filtering a mode of transport is fully expressed in a comprehensive model for laying out neighbourhoods and districts – the Fused Grid. +At present the A35 autoroute, which parallels the Rhine between Karlsruhe and Basel, and the A4 autoroute, which links Paris with Strasbourg, penetrate close to the centre of the city. The Grand contournement ouest (GCO) project, programmed since 1999, plans to construct a 24 km (15 mi) long highway connection between the junctions of the A4 and the A35 autoroutes in the north and of the A35 and A352 autoroutes in the south. This routes well to the west of the city and is meant to divest a significant portion of motorized traffic from the unité urbaine. diff --git a/e2e/samples/test_documents/War_on_Terror.txt b/e2e/samples/test_documents/War_on_Terror.txt new file mode 100644 index 0000000000..1601e747ba --- /dev/null +++ b/e2e/samples/test_documents/War_on_Terror.txt @@ -0,0 +1,31 @@ +On 16 September 2001, at Camp David, President George W. Bush used the phrase war on terrorism in an unscripted and controversial comment when he said, "This crusade – this war on terrorism – is going to take a while, ... " Bush later apologized for this remark due to the negative connotations the term crusade has to people, e.g. of Muslim faith. The word crusade was not used again. On 20 September 2001, during a televised address to a joint session of congress, Bush stated that, "(o)ur 'war on terror' begins with al-Qaeda, but it does not end there. It will not end until every terrorist group of global reach has been found, stopped, and defeated." +U.S. President Barack Obama has rarely used the term, but in his inaugural address on 20 January 2009, he stated "Our nation is at war, against a far-reaching network of violence and hatred." In March 2009 the Defense Department officially changed the name of operations from "Global War on Terror" to "Overseas Contingency Operation" (OCO). In March 2009, the Obama administration requested that Pentagon staff members avoid use of the term, instead using "Overseas Contingency Operation". Basic objectives of the Bush administration "war on terror", such as targeting al Qaeda and building international counterterrorism alliances, remain in place. In December 2012, Jeh Johnson, the General Counsel of the Department of Defense, stated that the military fight will be replaced by a law enforcement operation when speaking at Oxford University, predicting that al Qaeda will be so weakened to be ineffective, and has been "effectively destroyed", and thus the conflict will not be an armed conflict under international law. In May 2013, Obama stated that the goal is "to dismantle specific networks of violent extremists that threaten America"; which coincided with the U.S. Office of Management and Budget having changed the wording from "Overseas Contingency Operations" to "Countering Violent Extremism" in 2010. +Because the actions involved in the "war on terrorism" are diffuse, and the criteria for inclusion are unclear, political theorist Richard Jackson has argued that "the 'war on terrorism' therefore, is simultaneously a set of actual practices—wars, covert operations, agencies, and institutions—and an accompanying series of assumptions, beliefs, justifications, and narratives—it is an entire language or discourse." Jackson cites among many examples a statement by John Ashcroft that "the attacks of September 11 drew a bright line of demarcation between the civil and the savage". Administration officials also described "terrorists" as hateful, treacherous, barbarous, mad, twisted, perverted, without faith, parasitical, inhuman, and, most commonly, evil. Americans, in contrast, were described as brave, loving, generous, strong, resourceful, heroic, and respectful of human rights. +The origins of al-Qaeda can be traced to the Soviet war in Afghanistan (December 1979 – February 1989). The United States, United Kingdom, Saudi Arabia, Pakistan, and the People's Republic of China supported the Islamist Afghan mujahadeen guerillas against the military forces of the Soviet Union and the Democratic Republic of Afghanistan. A small number of "Afghan Arab" volunteers joined the fight against the Soviets, including Osama bin Laden, but there is no evidence they received any external assistance. In May 1996 the group World Islamic Front for Jihad Against Jews and Crusaders (WIFJAJC), sponsored by bin Laden (and later re-formed as al-Qaeda), started forming a large base of operations in Afghanistan, where the Islamist extremist regime of the Taliban had seized power earlier in the year. In February 1998, Osama bin Laden signed a fatwā, as head of al-Qaeda, declaring war on the West and Israel, later in May of that same year al-Qaeda released a video declaring war on the U.S. and the West. +On 7 August 1998, al-Qaeda struck the U.S. embassies in Kenya and Tanzania, killing 224 people, including 12 Americans. In retaliation, U.S. President Bill Clinton launched Operation Infinite Reach, a bombing campaign in Sudan and Afghanistan against targets the U.S. asserted were associated with WIFJAJC, although others have questioned whether a pharmaceutical plant in Sudan was used as a chemical warfare plant. The plant produced much of the region's antimalarial drugs and around 50% of Sudan's pharmaceutical needs. The strikes failed to kill any leaders of WIFJAJC or the Taliban. +On the morning of 11 September 2001, 19 men affiliated with al-Qaeda hijacked four airliners all bound for California. Once the hijackers assumed control of the airliners, they told the passengers that they had the bomb on board and would spare the lives of passengers and crew once their demands were met – no passenger and crew actually suspected that they would use the airliners as suicide weapons since it had never happened before in history. The hijackers – members of al-Qaeda's Hamburg cell – intentionally crashed two airliners into the Twin Towers of the World Trade Center in New York City. Both buildings collapsed within two hours from fire damage related to the crashes, destroying nearby buildings and damaging others. The hijackers crashed a third airliner into the Pentagon in Arlington County, Virginia, just outside Washington D.C. The fourth plane crashed into a field near Shanksville, Pennsylvania, after some of its passengers and flight crew attempted to retake control of the plane, which the hijackers had redirected toward Washington D.C., to target the White House, or the U.S. Capitol. No flights had survivors. A total of 2,977 victims and the 19 hijackers perished in the attacks. +The Authorization for Use of Military Force Against Terrorists or "AUMF" was made law on 14 September 2001, to authorize the use of United States Armed Forces against those responsible for the attacks on 11 September 2001. It authorized the President to use all necessary and appropriate force against those nations, organizations, or persons he determines planned, authorized, committed, or aided the terrorist attacks that occurred on 11 September 2001, or harbored such organizations or persons, in order to prevent any future acts of international terrorism against the United States by such nations, organizations or persons. Congress declares this is intended to constitute specific statutory authorization within the meaning of section 5(b) of the War Powers Resolution of 1973. +Subsequently, in October 2001, U.S. forces (with UK and coalition allies) invaded Afghanistan to oust the Taliban regime. On 7 October 2001, the official invasion began with British and U.S. forces conducting airstrike campaigns over enemy targets. Kabul, the capital city of Afghanistan, fell by mid-November. The remaining al-Qaeda and Taliban remnants fell back to the rugged mountains of eastern Afghanistan, mainly Tora Bora. In December, Coalition forces (the U.S. and its allies) fought within that region. It is believed that Osama bin Laden escaped into Pakistan during the battle. +The Taliban regrouped in western Pakistan and began to unleash an insurgent-style offensive against Coalition forces in late 2002. Throughout southern and eastern Afghanistan, firefights broke out between the surging Taliban and Coalition forces. Coalition forces responded with a series of military offensives and an increase in the amount of troops in Afghanistan. In February 2010, Coalition forces launched Operation Moshtarak in southern Afghanistan along with other military offensives in the hopes that they would destroy the Taliban insurgency once and for all. Peace talks are also underway between Taliban affiliated fighters and Coalition forces. In September 2014, Afghanistan and the United States signed a security agreement, which permits United States and NATO forces to remain in Afghanistan until at least 2024. The United States and other NATO and non-NATO forces are planning to withdraw; with the Taliban claiming it has defeated the United States and NATO, and the Obama Administration viewing it as a victory. In December 2014, ISAF encasing its colors, and Resolute Support began as the NATO operation in Afghanistan. Continued United States operations within Afghanistan will continue under the name "Operation Freedom's Sentinel". +In January 2002, the United States Special Operations Command, Pacific deployed to the Philippines to advise and assist the Armed Forces of the Philippines in combating Filipino Islamist groups. The operations were mainly focused on removing the Abu Sayyaf group and Jemaah Islamiyah (JI) from their stronghold on the island of Basilan. The second portion of the operation was conducted as a humanitarian program called "Operation Smiles". The goal of the program was to provide medical care and services to the region of Basilan as part of a "Hearts and Minds" program. Joint Special Operations Task Force – Philippines disbanded in June 2014, ending a 14-year mission. After JSOTF-P disbanded, as late as November 2014, American forces continued to operate in the Philippines under the name "PACOM Augmentation Team". +On 14 September 2009, U.S. Special Forces killed two men and wounded and captured two others near the Somali village of Baarawe. Witnesses claim that helicopters used for the operation launched from French-flagged warships, but that could not be confirmed. A Somali-based al-Qaida affiliated group, the Al-Shabaab, has confirmed the death of "sheik commander" Saleh Ali Saleh Nabhan along with an unspecified number of militants. Nabhan, a Kenyan, was wanted in connection with the 2002 Mombasa attacks. +The conflict in northern Mali began in January 2012 with radical Islamists (affiliated to al-Qaeda) advancing into northern Mali. The Malian government had a hard time maintaining full control over their country. The fledgling government requested support from the international community on combating the Islamic militants. In January 2013, France intervened on behalf of the Malian government's request and deployed troops into the region. They launched Operation Serval on 11 January 2013, with the hopes of dislodging the al-Qaeda affiliated groups from northern Mali. +Following the ceasefire agreement that suspended hostilities (but not officially ended) in the 1991 Gulf War, the United States and its allies instituted and began patrolling Iraqi no-fly zones, to protect Iraq's Kurdish and Shi'a Arab population—both of which suffered attacks from the Hussein regime before and after the Gulf War—in Iraq's northern and southern regions, respectively. U.S. forces continued in combat zone deployments through November 1995 and launched Operation Desert Fox against Iraq in 1998 after it failed to meet U.S. demands of "unconditional cooperation" in weapons inspections. +The first ground attack came at the Battle of Umm Qasr on 21 March 2003 when a combined force of British, American and Polish forces seized control of the port city of Umm Qasr. Baghdad, Iraq's capital city, fell to American forces in April 2003 and Saddam Hussein's government quickly dissolved. On 1 May 2003, Bush announced that major combat operations in Iraq had ended. However, an insurgency arose against the U.S.-led coalition and the newly developing Iraqi military and post-Saddam government. The insurgency, which included al-Qaeda affiliated groups, led to far more coalition casualties than the invasion. Other elements of the insurgency were led by fugitive members of President Hussein's Ba'ath regime, which included Iraqi nationalists and pan-Arabists. Many insurgency leaders are Islamists and claim to be fighting a religious war to reestablish the Islamic Caliphate of centuries past. Iraq's former president, Saddam Hussein was captured by U.S. forces in December 2003. He was executed in 2006. +In a major split in the ranks of Al Qaeda's organization, the Iraqi franchise, known as Al Qaeda in Iraq covertly invaded Syria and the Levant and began participating in the ongoing Syrian Civil War, gaining enough support and strength to re-invade Iraq's western provinces under the name of the Islamic State of Iraq and the Levant (ISIS/ISIL), taking over much of the country in a blitzkrieg-like action and combining the Iraq insurgency and Syrian Civil War into a single conflict. Due to their extreme brutality and a complete change in their overall ideology, Al Qaeda's core organization in Central Asia eventually denounced ISIS and directed their affiliates to cut off all ties with this organization. Many analysts[who?] believe that because of this schism, Al Qaeda and ISIL are now in a competition to retain the title of the world's most powerful terrorist organization. +The Obama administration began to reengage in Iraq with a series of airstrikes aimed at ISIS beginning on 10 August 2014. On 9 September 2014 President Obama said that he had the authority he needed to take action to destroy the militant group known as the Islamic State of Iraq and the Levant, citing the 2001 Authorization for Use of Military Force Against Terrorists, and thus did not require additional approval from Congress. The following day on 10 September 2014 President Barack Obama made a televised speech about ISIL, which he stated "Our objective is clear: We will degrade, and ultimately destroy, ISIL through a comprehensive and sustained counter-terrorism strategy". Obama has authorized the deployment of additional U.S. Forces into Iraq, as well as authorizing direct military operations against ISIL within Syria. On the night of 21/22 September the United States, Saudi Arabia, Bahrain, the UAE, Jordan and Qatar started air attacks against ISIS in Syria.[citation needed] +Following the 11 September 2001 attacks, former President of Pakistan Pervez Musharraf sided with the U.S. against the Taliban government in Afghanistan after an ultimatum by then U.S. President George W. Bush. Musharraf agreed to give the U.S. the use of three airbases for Operation Enduring Freedom. United States Secretary of State Colin Powell and other U.S. administration officials met with Musharraf. On 19 September 2001, Musharraf addressed the people of Pakistan and stated that, while he opposed military tactics against the Taliban, Pakistan risked being endangered by an alliance of India and the U.S. if it did not cooperate. In 2006, Musharraf testified that this stance was pressured by threats from the U.S., and revealed in his memoirs that he had "war-gamed" the United States as an adversary and decided that it would end in a loss for Pakistan. +On 12 January 2002, Musharraf gave a speech against Islamic extremism. He unequivocally condemned all acts of terrorism and pledged to combat Islamic extremism and lawlessness within Pakistan itself. He stated that his government was committed to rooting out extremism and made it clear that the banned militant organizations would not be allowed to resurface under any new name. He said, "the recent decision to ban extremist groups promoting militancy was taken in the national interest after thorough consultations. It was not taken under any foreign influence". +In 2002, the Musharraf-led government took a firm stand against the jihadi organizations and groups promoting extremism, and arrested Maulana Masood Azhar, head of the Jaish-e-Mohammed, and Hafiz Muhammad Saeed, chief of the Lashkar-e-Taiba, and took dozens of activists into custody. An official ban was imposed on the groups on 12 January. Later that year, the Saudi born Zayn al-Abidn Muhammed Hasayn Abu Zubaydah was arrested by Pakistani officials during a series of joint U.S.-Pakistan raids. Zubaydah is said to have been a high-ranking al-Qaeda official with the title of operations chief and in charge of running al-Qaeda training camps. Other prominent al-Qaeda members were arrested in the following two years, namely Ramzi bin al-Shibh, who is known to have been a financial backer of al-Qaeda operations, and Khalid Sheikh Mohammed, who at the time of his capture was the third highest-ranking official in al-Qaeda and had been directly in charge of the planning for the 11 September attacks. +The use of drones by the Central Intelligence Agency in Pakistan to carry out operations associated with the Global War on Terror sparks debate over sovereignty and the laws of war. The U.S. Government uses the CIA rather than the U.S. Air Force for strikes in Pakistan in order to avoid breaching sovereignty through military invasion. The United States was criticized by[according to whom?] a report on drone warfare and aerial sovereignty for abusing the term 'Global War on Terror' to carry out military operations through government agencies without formally declaring war. +In a 'Letter to American People' written by Osama bin Laden in 2002, he stated that one of the reasons he was fighting America is because of its support of India on the Kashmir issue. While on a trip to Delhi in 2002, U.S. Secretary of Defense Donald Rumsfeld suggested that Al-Qaeda was active in Kashmir, though he did not have any hard evidence. An investigation in 2002 unearthed evidence that Al-Qaeda and its affiliates were prospering in Pakistan-administered Kashmir with tacit approval of Pakistan's National Intelligence agency Inter-Services Intelligence. A team of Special Air Service and Delta Force was sent into Indian-administered Kashmir in 2002 to hunt for Osama bin Laden after reports that he was being sheltered by the Kashmiri militant group Harkat-ul-Mujahideen. U.S. officials believed that Al-Qaeda was helping organize a campaign of terror in Kashmir in order to provoke conflict between India and Pakistan. Fazlur Rehman Khalil, the leader of the Harkat-ul-Mujahideen, signed al-Qaeda's 1998 declaration of holy war, which called on Muslims to attack all Americans and their allies. Indian sources claimed that In 2006, Al-Qaeda claimed they had established a wing in Kashmir; this worried the Indian government. India also claimed that Al-Qaeda has strong ties with the Kashmir militant groups Lashkar-e-Taiba and Jaish-e-Mohammed in Pakistan. While on a visit to Pakistan in January 2010, U.S. Defense secretary Robert Gates stated that Al-Qaeda was seeking to destabilize the region and planning to provoke a nuclear war between India and Pakistan. +In September 2009, a U.S. Drone strike reportedly killed Ilyas Kashmiri, who was the chief of Harkat-ul-Jihad al-Islami, a Kashmiri militant group associated with Al-Qaeda. Kashmiri was described by Bruce Riedel as a 'prominent' Al-Qaeda member, while others described him as the head of military operations for Al-Qaeda. Waziristan had now become the new battlefield for Kashmiri militants, who were now fighting NATO in support of Al-Qaeda. On 8 July 2012, Al-Badar Mujahideen, a breakaway faction of Kashmir centric terror group Hizbul Mujahideen, on conclusion of their two-day Shuhada Conference called for mobilisation of resources for continuation of jihad in Kashmir. +In the following months, NATO took a wide range of measures to respond to the threat of terrorism. On 22 November 2002, the member states of the Euro-Atlantic Partnership Council (EAPC) decided on a Partnership Action Plan against Terrorism, which explicitly states, "EAPC States are committed to the protection and promotion of fundamental freedoms and human rights, as well as the rule of law, in combating terrorism." NATO started naval operations in the Mediterranean Sea designed to prevent the movement of terrorists or weapons of mass destruction as well as to enhance the security of shipping in general called Operation Active Endeavour. +Support for the U.S. cooled when America made clear its determination to invade Iraq in late 2002. Even so, many of the "coalition of the willing" countries that unconditionally supported the U.S.-led military action have sent troops to Afghanistan, particular neighboring Pakistan, which has disowned its earlier support for the Taliban and contributed tens of thousands of soldiers to the conflict. Pakistan was also engaged in the War in North-West Pakistan (Waziristan War). Supported by U.S. intelligence, Pakistan was attempting to remove the Taliban insurgency and al-Qaeda element from the northern tribal areas. +The British 16th Air Assault Brigade (later reinforced by Royal Marines) formed the core of the force in southern Afghanistan, along with troops and helicopters from Australia, Canada and the Netherlands. The initial force consisted of roughly 3,300 British, 2,000 Canadian, 1,400 from the Netherlands and 240 from Australia, along with special forces from Denmark and Estonia and small contingents from other nations. The monthly supply of cargo containers through Pakistani route to ISAF in Afghanistan is over 4,000 costing around 12 billion in Pakistani Rupees. +In addition to military efforts abroad, in the aftermath of 9/11 the Bush Administration increased domestic efforts to prevent future attacks. Various government bureaucracies that handled security and military functions were reorganized. A new cabinet-level agency called the United States Department of Homeland Security was created in November 2002 to lead and coordinate the largest reorganization of the U.S. federal government since the consolidation of the armed forces into the Department of Defense.[citation needed] +The USA PATRIOT Act of October 2001 dramatically reduces restrictions on law enforcement agencies' ability to search telephone, e-mail communications, medical, financial, and other records; eases restrictions on foreign intelligence gathering within the United States; expands the Secretary of the Treasury's authority to regulate financial transactions, particularly those involving foreign individuals and entities; and broadens the discretion of law enforcement and immigration authorities in detaining and deporting immigrants suspected of terrorism-related acts. The act also expanded the definition of terrorism to include domestic terrorism, thus enlarging the number of activities to which the USA PATRIOT Act's expanded law enforcement powers could be applied. A new Terrorist Finance Tracking Program monitored the movements of terrorists' financial resources (discontinued after being revealed by The New York Times). Global telecommunication usage, including those with no links to terrorism, is being collected and monitored through the NSA electronic surveillance program. The Patriot Act is still in effect. +Political interest groups have stated that these laws remove important restrictions on governmental authority, and are a dangerous encroachment on civil liberties, possible unconstitutional violations of the Fourth Amendment. On 30 July 2003, the American Civil Liberties Union (ACLU) filed the first legal challenge against Section 215 of the Patriot Act, claiming that it allows the FBI to violate a citizen's First Amendment rights, Fourth Amendment rights, and right to due process, by granting the government the right to search a person's business, bookstore, and library records in a terrorist investigation, without disclosing to the individual that records were being searched. Also, governing bodies in a number of communities have passed symbolic resolutions against the act. +In 2005, the UN Security Council adopted Resolution 1624 concerning incitement to commit acts of terrorism and the obligations of countries to comply with international human rights laws. Although both resolutions require mandatory annual reports on counter-terrorism activities by adopting nations, the United States and Israel have both declined to submit reports. In the same year, the United States Department of Defense and the Chairman of the Joint Chiefs of Staff issued a planning document, by the name "National Military Strategic Plan for the War on Terrorism", which stated that it constituted the "comprehensive military plan to prosecute the Global War on Terror for the Armed Forces of the United States...including the findings and recommendations of the 9/11 Commission and a rigorous examination with the Department of Defense". +Criticism of the War on Terror addresses the issues, morality, efficiency, economics, and other questions surrounding the War on Terror and made against the phrase itself, calling it a misnomer. The notion of a "war" against "terrorism" has proven highly contentious, with critics charging that it has been exploited by participating governments to pursue long-standing policy/military objectives, reduce civil liberties, and infringe upon human rights. It is argued that the term war is not appropriate in this context (as in War on Drugs), since there is no identifiable enemy, and that it is unlikely international terrorism can be brought to an end by military means. +Other critics, such as Francis Fukuyama, note that "terrorism" is not an enemy, but a tactic; calling it a "war on terror", obscures differences between conflicts such as anti-occupation insurgents and international mujahideen. With a military presence in Iraq and Afghanistan and its associated collateral damage Shirley Williams maintains this increases resentment and terrorist threats against the West. There is also perceived U.S. hypocrisy, media-induced hysteria, and that differences in foreign and security policy have damaged America's image in most of the world. From ae0ec5e297dacffad0cdc0a342ec8e721e08931e Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Fri, 26 Apr 2024 10:21:11 +0200 Subject: [PATCH 08/13] fixing tests --- e2e/pipelines/test_evaluation_pipeline.py | 44 ++++++++++++++++------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py index 3cff1a9bcb..e10ab9525f 100644 --- a/e2e/pipelines/test_evaluation_pipeline.py +++ b/e2e/pipelines/test_evaluation_pipeline.py @@ -28,7 +28,7 @@ def indexing_pipeline(documents: List[Document]): """Indexing the documents""" document_store = InMemoryDocumentStore() doc_writer = DocumentWriter(document_store=document_store, policy=DuplicatePolicy.SKIP) - doc_embedder = SentenceTransformersDocumentEmbedder(model=embeddings_model) + doc_embedder = SentenceTransformersDocumentEmbedder(model=embeddings_model, progress_bar=False) ingestion_pipe = Pipeline() ingestion_pipe.add_component(instance=doc_embedder, name="doc_embedder") ingestion_pipe.add_component(instance=doc_writer, name="doc_writer") @@ -38,7 +38,7 @@ def indexing_pipeline(documents: List[Document]): def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int): - """Building the RAG pipeline""" + """RAG pipeline""" template = """ You have to answer the following question based on the given context information only. @@ -51,7 +51,7 @@ def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int): Answer: """ rag = Pipeline() - rag.add_component("embedder", SentenceTransformersTextEmbedder(model=embeddings_model)) + rag.add_component("embedder", SentenceTransformersTextEmbedder(model=embeddings_model, progress_bar=False)) rag.add_component("retriever", InMemoryEmbeddingRetriever(document_store, top_k=top_k)) rag.add_component("prompt_builder", PromptBuilder(template=template)) rag.add_component("generator", OpenAIGenerator(model="gpt-3.5-turbo")) @@ -98,7 +98,7 @@ def run_rag_pipeline(documents, evaluation_questions, rag_pipeline_a): truth_docs = [] retrieved_docs = [] contexts = [] - pred_answers = [] + predicted_answers = [] for q in evaluation_questions: response = rag_pipeline_a.run( @@ -111,9 +111,9 @@ def run_rag_pipeline(documents, evaluation_questions, rag_pipeline_a): truth_docs.append([doc for doc in documents if doc.meta["name"] in q["ground_truth_doc"] and doc.content]) retrieved_docs.append(response["answer_builder"]["answers"][0].documents) contexts.append([doc.content for doc in response["answer_builder"]["answers"][0].documents]) - pred_answers.append(response["answer_builder"]["answers"][0].data) + predicted_answers.append(response["answer_builder"]["answers"][0].data) - return contexts, pred_answers, retrieved_docs, truth_docs + return contexts, predicted_answers, retrieved_docs, truth_docs @pytest.mark.skipif( @@ -121,10 +121,7 @@ def run_rag_pipeline(documents, evaluation_questions, rag_pipeline_a): reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.", ) def test_evaluation_pipeline(samples_path): - """ - Test custom built evaluation pipeline - """ - + """Test an evaluation pipeline""" eval_questions = [ { "question": 'What falls within the term "cultural anthropology"?', @@ -148,8 +145,8 @@ def test_evaluation_pipeline(samples_path): for article in os.listdir(full_path): with open(f"{full_path}/{article}", "r") as f: for text in f.read().split("\n"): - if text: - docs.append(Document(content=text, meta={"name": article})) + docs.append(Document(content=text, meta={"name": article})) if text else None + doc_store = indexing_pipeline(docs) questions = [q["question"] for q in eval_questions] @@ -196,6 +193,7 @@ def test_evaluation_pipeline(samples_path): evaluation_result_a = EvaluationRunResult(run_name="rag_pipeline_a", results=results_a, inputs=inputs_a) df_score_report = evaluation_result_a.score_report() + # assert the score report has all the metrics assert len(df_score_report) == 6 assert list(df_score_report.columns) == ["score"] assert list(df_score_report.index) == [ @@ -206,6 +204,8 @@ def test_evaluation_pipeline(samples_path): "Document Recall Single Hit", "Document Recall Multi Hit", ] + + # assert the evaluation result has all the metrics, inputs and questions df = evaluation_result_a.to_pandas() assert list(df.columns) == [ "question", @@ -261,4 +261,24 @@ def test_evaluation_pipeline(samples_path): } evaluation_result_b = EvaluationRunResult(run_name="rag_pipeline_b", results=results_b, inputs=inputs_b) df_comparative = evaluation_result_a.comparative_individual_scores_report(evaluation_result_b) + + # assert the comparative score report has all the metrics, inputs and questions assert len(df_comparative) == 3 + assert list(df_comparative.columns) == [ + "question", + "contexts", + "answer", + "predicted_answer", + "rag_pipeline_a_Mean Reciprocal Rank", + "rag_pipeline_a_Semantic Answer Similarity", + "rag_pipeline_a_Faithfulness", + "rag_pipeline_a_Document MAP", + "rag_pipeline_a_Document Recall Single Hit", + "rag_pipeline_a_Document Recall Multi Hit", + "rag_pipeline_b_Mean Reciprocal Rank", + "rag_pipeline_b_Semantic Answer Similarity", + "rag_pipeline_b_Faithfulness", + "rag_pipeline_b_Document MAP", + "rag_pipeline_b_Document Recall Single Hit", + "rag_pipeline_b_Document Recall Multi Hit", + ] From ec2214d3ee0bc5ac79b07e49991fd83fb41ee7fc Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Fri, 26 Apr 2024 11:12:13 +0200 Subject: [PATCH 09/13] adding context relevance --- e2e/pipelines/test_evaluation_pipeline.py | 119 ++++++++++------------ 1 file changed, 54 insertions(+), 65 deletions(-) diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py index e10ab9525f..51b91a635d 100644 --- a/e2e/pipelines/test_evaluation_pipeline.py +++ b/e2e/pipelines/test_evaluation_pipeline.py @@ -7,6 +7,7 @@ from haystack.components.builders import AnswerBuilder, PromptBuilder from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder from haystack.components.evaluators import ( + ContextRelevanceEvaluator, DocumentMAPEvaluator, DocumentMRREvaluator, DocumentRecallEvaluator, @@ -66,7 +67,7 @@ def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int): return rag -def evaluation_pipeline(questions, truth_docs, truth_answers, retrieved_docs, contexts, pred_answers): +def eval_pipeline(questions, truth_docs, truth_answers, retrieved_docs, contexts, pred_answers): """ Run the evaluation pipeline """ @@ -77,6 +78,7 @@ def evaluation_pipeline(questions, truth_docs, truth_answers, retrieved_docs, co eval_pipeline.add_component("doc_map", DocumentMAPEvaluator()) eval_pipeline.add_component("doc_recall_single_hit", DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT)) eval_pipeline.add_component("doc_recall_multi_hit", DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT)) + eval_pipeline.add_component("relevance", ContextRelevanceEvaluator()) return eval_pipeline.run( { @@ -86,6 +88,7 @@ def evaluation_pipeline(questions, truth_docs, truth_answers, retrieved_docs, co "doc_map": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, "doc_recall_single_hit": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, "doc_recall_multi_hit": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, + "relevance": {"questions": questions, "contexts": contexts}, } ) @@ -116,6 +119,40 @@ def run_rag_pipeline(documents, evaluation_questions, rag_pipeline_a): return contexts, predicted_answers, retrieved_docs, truth_docs +def built_input_for_results_eval(rag_results): + """Helper function to build the input for the results evaluation""" + return { + "Mean Reciprocal Rank": { + "individual_scores": rag_results["doc_mrr"]["individual_scores"], + "score": rag_results["doc_mrr"]["score"], + }, + "Semantic Answer Similarity": { + "individual_scores": rag_results["sas"]["individual_scores"], + "score": rag_results["sas"]["score"], + }, + "Faithfulness": { + "individual_scores": rag_results["groundness"]["individual_scores"], + "score": rag_results["groundness"]["score"], + }, + "Document MAP": { + "individual_scores": rag_results["doc_map"]["individual_scores"], + "score": rag_results["doc_map"]["score"], + }, + "Document Recall Single Hit": { + "individual_scores": rag_results["doc_recall_single_hit"]["individual_scores"], + "score": rag_results["doc_recall_single_hit"]["score"], + }, + "Document Recall Multi Hit": { + "individual_scores": rag_results["doc_recall_multi_hit"]["individual_scores"], + "score": rag_results["doc_recall_multi_hit"]["score"], + }, + "Contextual Relevance": { + "individual_scores": rag_results["relevance"]["individual_scores"], + "score": rag_results["relevance"]["score"], + }, + } + + @pytest.mark.skipif( not os.environ.get("OPENAI_API_KEY", None), reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.", @@ -140,23 +177,22 @@ def test_evaluation_pipeline(samples_path): }, ] + questions = [q["question"] for q in eval_questions] + truth_answers = [q["answer"] for q in eval_questions] + + # indexing documents docs = [] full_path = os.path.join(str(samples_path) + "/test_documents/") for article in os.listdir(full_path): with open(f"{full_path}/{article}", "r") as f: for text in f.read().split("\n"): docs.append(Document(content=text, meta={"name": article})) if text else None - doc_store = indexing_pipeline(docs) - questions = [q["question"] for q in eval_questions] - truth_answers = [q["answer"] for q in eval_questions] - + # running the RAG pipeline A rag_pipeline_a = rag_pipeline(doc_store, top_k=2) contexts_a, pred_answers_a, retrieved_docs_a, truth_docs = run_rag_pipeline(docs, eval_questions, rag_pipeline_a) - results_rag_a = evaluation_pipeline( - questions, truth_docs, truth_answers, retrieved_docs_a, contexts_a, pred_answers_a - ) + results_rag_a = eval_pipeline(questions, truth_docs, truth_answers, retrieved_docs_a, contexts_a, pred_answers_a) inputs_a = { "question": questions, @@ -164,37 +200,12 @@ def test_evaluation_pipeline(samples_path): "answer": truth_answers, "predicted_answer": pred_answers_a, } - results_a = { - "Mean Reciprocal Rank": { - "individual_scores": results_rag_a["doc_mrr"]["individual_scores"], - "score": results_rag_a["doc_mrr"]["score"], - }, - "Semantic Answer Similarity": { - "individual_scores": results_rag_a["sas"]["individual_scores"], - "score": results_rag_a["sas"]["score"], - }, - "Faithfulness": { - "individual_scores": results_rag_a["groundness"]["individual_scores"], - "score": results_rag_a["groundness"]["score"], - }, - "Document MAP": { - "individual_scores": results_rag_a["doc_map"]["individual_scores"], - "score": results_rag_a["doc_map"]["score"], - }, - "Document Recall Single Hit": { - "individual_scores": results_rag_a["doc_recall_single_hit"]["individual_scores"], - "score": results_rag_a["doc_recall_single_hit"]["score"], - }, - "Document Recall Multi Hit": { - "individual_scores": results_rag_a["doc_recall_multi_hit"]["individual_scores"], - "score": results_rag_a["doc_recall_multi_hit"]["score"], - }, - } + results_a = built_input_for_results_eval(results_rag_a) evaluation_result_a = EvaluationRunResult(run_name="rag_pipeline_a", results=results_a, inputs=inputs_a) df_score_report = evaluation_result_a.score_report() # assert the score report has all the metrics - assert len(df_score_report) == 6 + assert len(df_score_report) == 7 assert list(df_score_report.columns) == ["score"] assert list(df_score_report.index) == [ "Mean Reciprocal Rank", @@ -203,6 +214,7 @@ def test_evaluation_pipeline(samples_path): "Document MAP", "Document Recall Single Hit", "Document Recall Multi Hit", + "Contextual Relevance", ] # assert the evaluation result has all the metrics, inputs and questions @@ -218,47 +230,22 @@ def test_evaluation_pipeline(samples_path): "Document MAP", "Document Recall Single Hit", "Document Recall Multi Hit", + "Contextual Relevance", ] assert len(df) == 3 + # running the RAG pipeline B rag_pipeline_b = rag_pipeline(doc_store, top_k=4) contexts_b, pred_answers_b, retrieved_docs_b, truth_docs = run_rag_pipeline(docs, eval_questions, rag_pipeline_b) - results_rag_b = evaluation_pipeline( - questions, truth_docs, truth_answers, retrieved_docs_b, contexts_b, pred_answers_b - ) + results_rag_b = eval_pipeline(questions, truth_docs, truth_answers, retrieved_docs_b, contexts_b, pred_answers_b) inputs_b = { "question": questions, - "contexts": contexts_a, + "contexts": contexts_b, "answer": truth_answers, "predicted_answer": pred_answers_b, } - results_b = { - "Mean Reciprocal Rank": { - "individual_scores": results_rag_b["doc_mrr"]["individual_scores"], - "score": results_rag_b["doc_mrr"]["score"], - }, - "Semantic Answer Similarity": { - "individual_scores": results_rag_b["sas"]["individual_scores"], - "score": results_rag_b["sas"]["score"], - }, - "Faithfulness": { - "individual_scores": results_rag_b["groundness"]["individual_scores"], - "score": results_rag_b["groundness"]["score"], - }, - "Document MAP": { - "individual_scores": results_rag_b["doc_map"]["individual_scores"], - "score": results_rag_b["doc_map"]["score"], - }, - "Document Recall Single Hit": { - "individual_scores": results_rag_b["doc_recall_single_hit"]["individual_scores"], - "score": results_rag_b["doc_recall_single_hit"]["score"], - }, - "Document Recall Multi Hit": { - "individual_scores": results_rag_b["doc_recall_multi_hit"]["individual_scores"], - "score": results_rag_b["doc_recall_multi_hit"]["score"], - }, - } + results_b = built_input_for_results_eval(results_rag_b) evaluation_result_b = EvaluationRunResult(run_name="rag_pipeline_b", results=results_b, inputs=inputs_b) df_comparative = evaluation_result_a.comparative_individual_scores_report(evaluation_result_b) @@ -275,10 +262,12 @@ def test_evaluation_pipeline(samples_path): "rag_pipeline_a_Document MAP", "rag_pipeline_a_Document Recall Single Hit", "rag_pipeline_a_Document Recall Multi Hit", + "rag_pipeline_a_Contextual Relevance", "rag_pipeline_b_Mean Reciprocal Rank", "rag_pipeline_b_Semantic Answer Similarity", "rag_pipeline_b_Faithfulness", "rag_pipeline_b_Document MAP", "rag_pipeline_b_Document Recall Single Hit", "rag_pipeline_b_Document Recall Multi Hit", + "rag_pipeline_b_Contextual Relevance", ] From 5d42e74ff8952847afba6b65845fea589df73ae4 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Fri, 26 Apr 2024 11:46:59 +0200 Subject: [PATCH 10/13] reverting some wrong changes to due PyCharm error in refactoring --- haystack/components/generators/chat/openai.py | 2 +- .../embedders/test_sentence_transformers_embedding_backend.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/haystack/components/generators/chat/openai.py b/haystack/components/generators/chat/openai.py index 5a71ac595f..8369892191 100644 --- a/haystack/components/generators/chat/openai.py +++ b/haystack/components/generators/chat/openai.py @@ -250,7 +250,7 @@ def _connect_chunks(self, chunk: Any, chunks: List[StreamingChunk]) -> ChatMessa complete_response = ChatMessage.from_assistant("".join([chunk.content for chunk in chunks])) complete_response.meta.update( { - "model": chunk.embeddings_model, + "model": chunk.model, "index": 0, "finish_reason": chunk.choices[0].finish_reason, "usage": {}, # we don't have usage data for streaming responses diff --git a/test/components/embedders/test_sentence_transformers_embedding_backend.py b/test/components/embedders/test_sentence_transformers_embedding_backend.py index a10acf9200..cbdc812ac7 100644 --- a/test/components/embedders/test_sentence_transformers_embedding_backend.py +++ b/test/components/embedders/test_sentence_transformers_embedding_backend.py @@ -39,4 +39,4 @@ def test_embedding_function_with_kwargs(mock_sentence_transformer): data = ["sentence1", "sentence2"] embedding_backend.embed(data=data, normalize_embeddings=True) - embedding_backend.embeddings_model.encode.assert_called_once_with(data, normalize_embeddings=True) + embedding_backend.model.encode.assert_called_once_with(data, normalize_embeddings=True) From 78c2157c849bb55266245a65894e356fa8bb73da Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Fri, 26 Apr 2024 11:56:03 +0200 Subject: [PATCH 11/13] reverting some wrong changes to due PyCharm error in refactoring --- proposals/text/4284-drop-basecomponent.md | 179 +++++++++++----------- 1 file changed, 87 insertions(+), 92 deletions(-) diff --git a/proposals/text/4284-drop-basecomponent.md b/proposals/text/4284-drop-basecomponent.md index 7fc5c18095..8ed961b6cb 100644 --- a/proposals/text/4284-drop-basecomponent.md +++ b/proposals/text/4284-drop-basecomponent.md @@ -533,108 +533,103 @@ A Haystack node is any class that abides the following contract: @node class MyNode: - def __init__(self, model_name: str: "deepset-ai/a-model-name" - - ): - """ - Haystack nodes should have an `__init__` method where they define: - - - `self.inputs = []`: - A list with all the edges they can possibly receive input from - - - `self.outputs = []`: - A list with the edges they might possibly produce as output - - - `self.init_parameters = {}`: - Any state they wish to be persisted in their YAML serialization. - These values will be given to the `__init__` method of a new instance - when the pipeline is deserialized. - - The `__init__` must be extremely lightweight, because it's a frequent - operation during the construction and validation of the pipeline. If a node - has some heavy state to initialize (models, backends, etc...) refer to the - `warm_up()` method. - """ - # Lightweight state can be initialized here, for example storing the model name - # to be loaded later. See self.warm_up() - self.embeddings_model = None - self.model_name = model_name - self.how_many_times_have_I_been_called = 0 - - # Contract - all three are mandatory. - self.init_parameters = {"model_name": model_name} - self.inputs = ["input_name"] - self.outputs = ["output_name"] + def __init__(self, model_name: str: "deepset-ai/a-model-name"): + """ + Haystack nodes should have an `__init__` method where they define: + + - `self.inputs = []`: + A list with all the edges they can possibly receive input from + + - `self.outputs = []`: + A list with the edges they might possibly produce as output + + - `self.init_parameters = {}`: + Any state they wish to be persisted in their YAML serialization. + These values will be given to the `__init__` method of a new instance + when the pipeline is deserialized. + + The `__init__` must be extremely lightweight, because it's a frequent + operation during the construction and validation of the pipeline. If a node + has some heavy state to initialize (models, backends, etc...) refer to the + `warm_up()` method. + """ + # Lightweight state can be initialized here, for example storing the model name + # to be loaded later. See self.warm_up() + self.model = None + self.model_name = model_name + self.how_many_times_have_I_been_called = 0 + # Contract - all three are mandatory. + self.init_parameters = {"model_name": model_name} + self.inputs = ["input_name"] + self.outputs = ["output_name"] -def warm_up(self): - """ - Optional method. - - This method is called by Pipeline before the graph execution. - Make sure to avoid double-initializations, because Pipeline will not keep - track of which nodes it called `warm_up` on. - """ - if not self.embeddings_model: - self.embeddings_model = AutoModel.load_from_pretrained(self.model_name) + def warm_up(self): + """ + Optional method. + This method is called by Pipeline before the graph execution. + Make sure to avoid double-initializations, because Pipeline will not keep + track of which nodes it called `warm_up` on. + """ + if not self.model: + self.model = AutoModel.load_from_pretrained(self.model_name) -def run( + def run( self, name: str, data: List[Tuple[str, Any]], parameters: Dict[str, Any], stores: Dict[str, Any], -): - """ - Mandatory method. - - This is the method where the main functionality of the node should be carried out. - It's called by `Pipeline.run()`, which passes the following parameters to it: - - - `name: str`: the name of the node. Allows the node to find its own parameters in the `parameters` dictionary (see below). - - - `data: List[Tuple[str, Any]]`: the input data. - Pipeline guarantees that the following assert always passes: `assert self.inputs == [name for name, value in data]`, - which means that: - - `data` is of the same length as `self.inputs`. - - `data` contains one tuple for each string stored in `self.inputs`. - - no guarantee is given on the values of these tuples: notably, if there was a decision node upstream, some values might be `None`. - For example, if a node declares `self.inputs = ["value", "value"]` (think of a Sum node), `data` might look like: - - `[("value", 1), ("value", 10)]` - - `[("value", None), ("value", 10)]` - - `[("value", None), ("value", None)]`, or even - - `[("value", 1), ("value", ["something", "unexpected"])]` - but it will never look like: - - `[("value", 1), ("value", 10), ("value", 100)]`, - - `[("value": 15)]` or - - `[("value": 15), ("unexpected", 10)]`. - - - `parameters: Dict[str, Dict[str, Any]]`: a dictionary of dictionaries with all the parameters for all nodes. - Note that all nodes have access to all parameters for all other nodes: this might come handy to nodes like `Agent`s, that - want to influence the behavior of nodes downstream. - Nodes can access their own parameters using `name`, but they must not assume their name is present in the dictionary. - Therefore the best way to get the parameters is with `my_parameters = parameters.get(name, {})` - - - `stores`: a dictionary of all the (Document)Stores connected to this pipeline. - - Pipeline expect the output of this function to be a tuple in the following format: - - `( {edge: value for edge in }, {the parameters dictionary}) - - Which means that: - - Nodes are not forced to produce output on all the expected outputs: for example nodes taking a decision, like classifiers, - can produce output on a subset of the expected output edges and Pipeline will figure out the rest. - - Nodes must not add any key in the data dictionary that is not present in `self.outputs`, - - Nodes can alter the content of `parameters` and their changes will be propagated downstream. - """ - self.how_many_times_have_I_been_called += 1 - - value = data[0][1] - print( - f"Hello I'm {name}! This instance have been called {self.how_many_times_have_I_been_called} times and this is the value I received: {value}") + ): + """ + Mandatory method. + + This is the method where the main functionality of the node should be carried out. + It's called by `Pipeline.run()`, which passes the following parameters to it: + + - `name: str`: the name of the node. Allows the node to find its own parameters in the `parameters` dictionary (see below). + + - `data: List[Tuple[str, Any]]`: the input data. + Pipeline guarantees that the following assert always passes: `assert self.inputs == [name for name, value in data]`, + which means that: + - `data` is of the same length as `self.inputs`. + - `data` contains one tuple for each string stored in `self.inputs`. + - no guarantee is given on the values of these tuples: notably, if there was a decision node upstream, some values might be `None`. + For example, if a node declares `self.inputs = ["value", "value"]` (think of a Sum node), `data` might look like: + - `[("value", 1), ("value", 10)]` + - `[("value", None), ("value", 10)]` + - `[("value", None), ("value", None)]`, or even + - `[("value", 1), ("value", ["something", "unexpected"])]` + but it will never look like: + - `[("value", 1), ("value", 10), ("value", 100)]`, + - `[("value": 15)]` or + - `[("value": 15), ("unexpected", 10)]`. + + - `parameters: Dict[str, Dict[str, Any]]`: a dictionary of dictionaries with all the parameters for all nodes. + Note that all nodes have access to all parameters for all other nodes: this might come handy to nodes like `Agent`s, that + want to influence the behavior of nodes downstream. + Nodes can access their own parameters using `name`, but they must not assume their name is present in the dictionary. + Therefore the best way to get the parameters is with `my_parameters = parameters.get(name, {})` + + - `stores`: a dictionary of all the (Document)Stores connected to this pipeline. + + Pipeline expect the output of this function to be a tuple in the following format: + + `( {edge: value for edge in }, {the parameters dictionary}) + + Which means that: + - Nodes are not forced to produce output on all the expected outputs: for example nodes taking a decision, like classifiers, + can produce output on a subset of the expected output edges and Pipeline will figure out the rest. + - Nodes must not add any key in the data dictionary that is not present in `self.outputs`, + - Nodes can alter the content of `parameters` and their changes will be propagated downstream. + """ + self.how_many_times_have_I_been_called += 1 + + value = data[0][1] + print(f"Hello I'm {name}! This instance have been called {self.how_many_times_have_I_been_called} times and this is the value I received: {value}") - return ({self.outputs[0]: value}, parameters) + return ({self.outputs[0]: value}, parameters) ``` This contract is stored in the docstring of `@node` and acts as the single source of truth. From e687144fd453d72aff9d8634308712573ef662e6 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Fri, 26 Apr 2024 12:57:00 +0200 Subject: [PATCH 12/13] building eval pipeline only once --- e2e/pipelines/test_evaluation_pipeline.py | 56 ++++++++++++++--------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py index 51b91a635d..f3b010fae9 100644 --- a/e2e/pipelines/test_evaluation_pipeline.py +++ b/e2e/pipelines/test_evaluation_pipeline.py @@ -22,14 +22,14 @@ from haystack.document_stores.types import DuplicatePolicy from haystack.evaluation import EvaluationRunResult -embeddings_model = "sentence-transformers/all-MiniLM-L6-v2" +EMBEDDINGS_MODEL = "sentence-transformers/all-MiniLM-L6-v2" def indexing_pipeline(documents: List[Document]): """Indexing the documents""" document_store = InMemoryDocumentStore() doc_writer = DocumentWriter(document_store=document_store, policy=DuplicatePolicy.SKIP) - doc_embedder = SentenceTransformersDocumentEmbedder(model=embeddings_model, progress_bar=False) + doc_embedder = SentenceTransformersDocumentEmbedder(model=EMBEDDINGS_MODEL, progress_bar=False) ingestion_pipe = Pipeline() ingestion_pipe.add_component(instance=doc_embedder, name="doc_embedder") ingestion_pipe.add_component(instance=doc_writer, name="doc_writer") @@ -38,7 +38,7 @@ def indexing_pipeline(documents: List[Document]): return document_store -def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int): +def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int): # type: ignore """RAG pipeline""" template = """ You have to answer the following question based on the given context information only. @@ -52,7 +52,7 @@ def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int): Answer: """ rag = Pipeline() - rag.add_component("embedder", SentenceTransformersTextEmbedder(model=embeddings_model, progress_bar=False)) + rag.add_component("embedder", SentenceTransformersTextEmbedder(model=EMBEDDINGS_MODEL, progress_bar=False)) rag.add_component("retriever", InMemoryEmbeddingRetriever(document_store, top_k=top_k)) rag.add_component("prompt_builder", PromptBuilder(template=template)) rag.add_component("generator", OpenAIGenerator(model="gpt-3.5-turbo")) @@ -67,30 +67,40 @@ def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int): return rag -def eval_pipeline(questions, truth_docs, truth_answers, retrieved_docs, contexts, pred_answers): +def evaluation_pipeline(): """ - Run the evaluation pipeline + Create an evaluation pipeline with the following evaluators: + + - DocumentMRREvaluator + - FaithfulnessEvaluator + - SASEvaluator + - DocumentMAPEvaluator + - DocumentRecallEvaluator + - ContextRelevanceEvaluator """ eval_pipeline = Pipeline() eval_pipeline.add_component("doc_mrr", DocumentMRREvaluator()) eval_pipeline.add_component("groundness", FaithfulnessEvaluator()) - eval_pipeline.add_component("sas", SASEvaluator(model=embeddings_model)) + eval_pipeline.add_component("sas", SASEvaluator(model=EMBEDDINGS_MODEL)) eval_pipeline.add_component("doc_map", DocumentMAPEvaluator()) eval_pipeline.add_component("doc_recall_single_hit", DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT)) eval_pipeline.add_component("doc_recall_multi_hit", DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT)) eval_pipeline.add_component("relevance", ContextRelevanceEvaluator()) - return eval_pipeline.run( - { - "doc_mrr": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, - "groundness": {"questions": questions, "contexts": contexts, "responses": truth_answers}, - "sas": {"predicted_answers": pred_answers, "ground_truth_answers": truth_answers}, - "doc_map": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, - "doc_recall_single_hit": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, - "doc_recall_multi_hit": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, - "relevance": {"questions": questions, "contexts": contexts}, - } - ) + return eval_pipeline + + +def built_eval_input(questions, truth_docs, truth_answers, retrieved_docs, contexts, pred_answers): + """Helper function to build the input for the evaluation pipeline""" + return { + "doc_mrr": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, + "groundness": {"questions": questions, "contexts": contexts, "responses": truth_answers}, + "sas": {"predicted_answers": pred_answers, "ground_truth_answers": truth_answers}, + "doc_map": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, + "doc_recall_single_hit": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, + "doc_recall_multi_hit": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs}, + "relevance": {"questions": questions, "contexts": contexts}, + } def run_rag_pipeline(documents, evaluation_questions, rag_pipeline_a): @@ -189,11 +199,14 @@ def test_evaluation_pipeline(samples_path): docs.append(Document(content=text, meta={"name": article})) if text else None doc_store = indexing_pipeline(docs) - # running the RAG pipeline A + # running the RAG pipeline A + evaluation pipeline rag_pipeline_a = rag_pipeline(doc_store, top_k=2) contexts_a, pred_answers_a, retrieved_docs_a, truth_docs = run_rag_pipeline(docs, eval_questions, rag_pipeline_a) - results_rag_a = eval_pipeline(questions, truth_docs, truth_answers, retrieved_docs_a, contexts_a, pred_answers_a) + eval_pipeline = evaluation_pipeline() + eval_input = built_eval_input(questions, truth_docs, truth_answers, retrieved_docs_a, contexts_a, pred_answers_a) + results_rag_a = eval_pipeline.run(eval_input) + # running the evaluation EvaluationRunResult inputs_a = { "question": questions, "contexts": contexts_a, @@ -237,7 +250,8 @@ def test_evaluation_pipeline(samples_path): # running the RAG pipeline B rag_pipeline_b = rag_pipeline(doc_store, top_k=4) contexts_b, pred_answers_b, retrieved_docs_b, truth_docs = run_rag_pipeline(docs, eval_questions, rag_pipeline_b) - results_rag_b = eval_pipeline(questions, truth_docs, truth_answers, retrieved_docs_b, contexts_b, pred_answers_b) + eval_input = built_eval_input(questions, truth_docs, truth_answers, retrieved_docs_b, contexts_b, pred_answers_b) + results_rag_b = eval_pipeline.run(eval_input) inputs_b = { "question": questions, From f3aeb428d3011154399a118ed563c33f871a5d36 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Fri, 26 Apr 2024 13:10:58 +0200 Subject: [PATCH 13/13] handling mypy issues --- e2e/pipelines/test_evaluation_pipeline.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py index f3b010fae9..1ea13f4148 100644 --- a/e2e/pipelines/test_evaluation_pipeline.py +++ b/e2e/pipelines/test_evaluation_pipeline.py @@ -31,8 +31,8 @@ def indexing_pipeline(documents: List[Document]): doc_writer = DocumentWriter(document_store=document_store, policy=DuplicatePolicy.SKIP) doc_embedder = SentenceTransformersDocumentEmbedder(model=EMBEDDINGS_MODEL, progress_bar=False) ingestion_pipe = Pipeline() - ingestion_pipe.add_component(instance=doc_embedder, name="doc_embedder") - ingestion_pipe.add_component(instance=doc_writer, name="doc_writer") + ingestion_pipe.add_component(instance=doc_embedder, name="doc_embedder") # type: ignore + ingestion_pipe.add_component(instance=doc_writer, name="doc_writer") # type: ignore ingestion_pipe.connect("doc_embedder.documents", "doc_writer.documents") ingestion_pipe.run({"doc_embedder": {"documents": documents}}) return document_store @@ -52,11 +52,11 @@ def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int): # type: ig Answer: """ rag = Pipeline() - rag.add_component("embedder", SentenceTransformersTextEmbedder(model=EMBEDDINGS_MODEL, progress_bar=False)) - rag.add_component("retriever", InMemoryEmbeddingRetriever(document_store, top_k=top_k)) - rag.add_component("prompt_builder", PromptBuilder(template=template)) - rag.add_component("generator", OpenAIGenerator(model="gpt-3.5-turbo")) - rag.add_component("answer_builder", AnswerBuilder()) + rag.add_component("embedder", SentenceTransformersTextEmbedder(model=EMBEDDINGS_MODEL, progress_bar=False)) # type: ignore + rag.add_component("retriever", InMemoryEmbeddingRetriever(document_store, top_k=top_k)) # type: ignore + rag.add_component("prompt_builder", PromptBuilder(template=template)) # type: ignore + rag.add_component("generator", OpenAIGenerator(model="gpt-3.5-turbo")) # type: ignore + rag.add_component("answer_builder", AnswerBuilder()) # type: ignore rag.connect("embedder", "retriever.query_embedding") rag.connect("retriever", "prompt_builder.documents") rag.connect("prompt_builder", "generator")