Data

.github/actions/ParlaMintValidate/action.yml

@@ -6,10 +6,13 @@ inputs:
     required: true
   requireTaxonomiesTranslations:
     description: 'require every term in common taxonomies to be translated'
-    default: '1'
+    required: false
 runs:
   using: "composite"
   steps:
+    - name: Set environment variables
+      run: echo "REQ_TRANSLATION=${{ inputs.requireTaxonomiesTranslations || '1' }}" >> $GITHUB_ENV
+      shell: bash
     - name: Convert and Validate
-      run: ${{ github.action_path }}/validate.sh '${{inputs.parlas}}' '${{inputs.requireTaxonomiesTranslations}}'
+      run: ${{ github.action_path }}/validate.sh '${{inputs.parlas}}' '${{env.REQ_TRANSLATION}}'
       shell: bash

.github/workflows/createSample.yml

@@ -49,16 +49,17 @@ jobs:
       - name: Create Parliaments Samples ${{needs.Changes.outputs.parla_changed}}
         run: |
           cd $GITHUB_WORKSPACE/ParlaMint
+          SAXON=Scripts/bin/saxon.jar
           for parla in $(jq -r '.[]' <<< '${{needs.Changes.outputs.parla_changed}}' ); do
             echo "::group::Processing ParlaMint-$parla"
             DIR="${{env.SAMPLE_DIR}}/$parla"
             mkdir $DIR
             echo "::notice::New sample files [$parla] TEXT"
-            java -jar $GITHUB_WORKSPACE/Saxon.jar outDir=$DIR revRespPers='GitHub Action' -xsl:${{env.SAMPLE_SCRIPT}} Samples/ParlaMint-$parla/ParlaMint-$parla.xml
+            java -jar $SAXON outDir=$DIR revRespPers='GitHub Action' -xsl:${{env.SAMPLE_SCRIPT}} Samples/ParlaMint-$parla/ParlaMint-$parla.xml
 
             echo "::notice::New sample files [$parla] ANNOTATED"
             if [ -f "Samples/ParlaMint-$parla/ParlaMint-$parla.ana.xml" ] ; then
-              java -jar $GITHUB_WORKSPACE/Saxon.jar outDir=$DIR revRespPers='GitHub Action' -xsl:${{env.SAMPLE_SCRIPT}} Samples/ParlaMint-$parla/ParlaMint-$parla.ana.xml
+              java -jar $SAXON outDir=$DIR revRespPers='GitHub Action' -xsl:${{env.SAMPLE_SCRIPT}} Samples/ParlaMint-$parla/ParlaMint-$parla.ana.xml
             else
               echo "::warning::skipping annotated conversion - missing corpus root file"
             fi

.gitignore

@@ -4,3 +4,10 @@ nohup.*
 *.zip
 *.tar
 *.tgz
+.idea/
+/output.log
+/output1.log
+/Samples/ParlaMint-IL/add-common-content/
+/Samples/ParlaMint-IL/text.seg/
+/Samples/ParlaMint-IL/text.seg.ana/
+/Samples/ParlaMint-IL/text.seg.ana/

Makefile

@@ -356,7 +356,7 @@ text: $(text-XX)
 $(text-XX): text-%: %
 	rm -f `ls ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/ParlaMint-$<_*.txt |  grep -v '.ana.'`
 	find -H ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX} -maxdepth 2 -type f -name "ParlaMint-$<_*.xml" | grep -v '.ana.' | $P --jobs 10 \
-	'$s -xsl:Scripts/parlamint-tei2text.xsl {} > ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/{/.}.txt'
+	'$s -xsl:Scripts/parlamint-tei2text.xsl {} > ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/{.}.txt'
 
 text.ana-XX = $(addprefix text.ana-, $(PARLIAMENTS))
 ## text.ana ## create text version from TEI.ana files
@@ -365,7 +365,7 @@ text.ana: $(text.ana-XX)
 $(text.ana-XX): text.ana-%: %
 	rm -f ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/ParlaMint-$<_*.ana.txt
 	find -H ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX} -maxdepth 2 -type f -name "ParlaMint-$<_*.xml" | grep '.ana.' | $P --jobs 10 \
-	'$s -xsl:Scripts/parlamint-tei2text.xsl {} > ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/{/.}.txt'
+	'$s -xsl:Scripts/parlamint-tei2text.xsl {} > ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/{.}.txt'
 
 
 
@@ -377,7 +377,7 @@ $(meta-XX): meta-%: %
 	rm -f ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/*-meta.tsv
 	find -H ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX} -maxdepth 2 -type f -name "ParlaMint-*_*.xml" | grep -v '.ana.' | $P --jobs 10 \
 	'$s meta=../${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/ParlaMint-$<.xml -xsl:Scripts/parlamint2meta.xsl \
-	{} > ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/{/.}-meta.tsv'
+	{} > ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/{.}-meta.tsv'
 
 
 
@@ -495,22 +495,27 @@ $(composite-teiHeader-INPLACE-XX): composite-teiHeader-INPLACE-%: % composite-te
 text.seg-XX = $(addprefix text.seg-, $(PARLIAMENTS))
 ## text.seg ## create text version from TEI files - each line contains one segment
 text.seg: $(text.seg-XX)
-## text-XX ## convert TEI files to text
+## text.seg-XX ## convert TEI files to text
 $(text.seg-XX): text.seg-%: %
-	mkdir -p ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg
-	rm -f `ls ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg/ParlaMint-$<_*.seg.txt |  grep -v '.ana.'`
-	find -H ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX} -maxdepth 2 -type f -name "ParlaMint-$<_*.xml" | grep -v '.ana.' | $P --jobs 10 \
-	'$s -xsl:Scripts/parlamint-tei2text.xsl element=seg {} > ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg/{/.}.txt'
+	@mkdir -p ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg
+	@rm -f ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg/ParlaMint-$<_*.seg.txt
+	@find -H ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX} -maxdepth 2 -type f -name "ParlaMint-$<_*.xml" | grep -v '.ana.' | $P --jobs 10 \
+	'$s -xsl:Scripts/parlamint-tei2text.xsl element=seg {} > ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg/{/.}.seg.txt'
+	@echo "INFO: segments converted to text are stored in ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg"
 
 text.seg.ana-XX = $(addprefix text.seg.ana-, $(PARLIAMENTS))
-## text.seg ## create text version from TEI.ana files - each line contains one segment
+## text.seg.ana ## create text version from TEI.ana files - each line contains one segment
 text.seg.ana: $(text.seg.ana-XX)
-## text.seg.ana-XX ## convert TEI.seg.ana files to text
+## text.seg.ana-XX ## convert TEI.ana files to text
 $(text.seg.ana-XX): text.seg.ana-%: %
-	mkdir -p ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg
-	rm -f ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg/ParlaMint-$<_*.seg.ana.txt
-	find -H ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX} -maxdepth 2 -type f -name "ParlaMint-$<_*.xml" | grep '.ana.' | $P --jobs 10 \
-	'$s -xsl:Scripts/parlamint-tei2text.xsl element=seg {} > ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg/{/.}.txt'
+	@mkdir -p ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg.ana
+	@rm -f ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg.ana/ParlaMint-$<_*.seg.txt
+	@find -H ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX} -maxdepth 2 -type f -name "ParlaMint-$<_*.xml" | grep '.ana.' | $P --jobs 10 \
+	'$s -xsl:Scripts/parlamint-tei2text.xsl element=seg {} > ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg.ana/{/.}'
+	@find -H ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg.ana -type f | $P  'mv {} {.}.seg.txt'
+	@echo "INFO: annotated segments converted to text are stored in ${DATADIR}/ParlaMint-$<${CORPUSDIR_SUFFIX}/text.seg.ana"
+
+
 
 
 ######---------------