Added subjectareas to the GYS and VUXGYS schooltypes to that the stru…

…cture aligns with GRS.
ist-group · Nov 29, 2019 · 1cc70c3 · 1cc70c3
1 parent b433455
commit 1cc70c3
Show file tree

Hide file tree

Showing 7 changed files with 105 additions and 35 deletions.
diff --git a/build.gradle b/build.gradle
@@ -5,7 +5,7 @@ plugins {
 }
 
 group = "com.github.stefan-jonasson"
-version = "0.7.4"
+version = "0.7.5"
 sourceCompatibility = 1.8
 
 repositories {

diff --git a/src/main/kotlin/org/edtech/curriculum/SchoolType.kt b/src/main/kotlin/org/edtech/curriculum/SchoolType.kt
@@ -3,7 +3,7 @@ package org.edtech.curriculum
 import java.io.*
 import java.net.URL
 
-enum class SchoolType(val filename: String, val archivePath: String) {
+enum class SchoolType(val filename: String, val archivePath: String, val archivePathSubjectArea: String? = null ) {
 
     /** https://www.skolverket.se/undervisning/grundskolan/laroplan-och-kursplaner-for-grundskolan */
     GR("compulsory.tgz", "compulsory/subject-compulsory-S2_0/grundskolan/"),
@@ -33,16 +33,13 @@ enum class SchoolType(val filename: String, val archivePath: String) {
     GY("syllabus.tgz", "subject/"),
 
     /** https://www.skolverket.se/undervisning/gymnasiesarskolan/laroplan-program-och-amnen-i-gymnasiesarskolan */
-    GYS("gys.tgz", "subject/"),
-
-    /** https://www.skolverket.se/undervisning/gymnasiesarskolan/laroplan-program-och-amnen-i-gymnasiesarskolan */
-    GYS_SUBJECT_AREA("gys.tgz", "subjectArea/"),
+    GYS("gys.tgz", "subject/", "subjectArea/"),
 
     /** https://www.skolverket.se/undervisning/vuxenutbildningen/komvux-grundlaggande */
     VUXGR("vuxgr.tgz", "subject/"),
 
     /** https://www.skolverket.se/undervisning/vuxenutbildningen/sarvux-grundlaggande */
-    VUXGRS("sarvuxgr.tgz", "subject/"),
+    VUXGRS("sarvuxgr.tgz", "subject/", "subjectArea/"),
 
     /** https://www.skolverket.se/undervisning/laroplaner-amnen-och-kurser/vuxenutbildning/komvux/sfi */
     SFI( "sfi.tgz", "subject/");

diff --git a/src/main/kotlin/org/edtech/curriculum/internal/IndividualFiledSubjectDataExtractor.kt b/src/main/kotlin/org/edtech/curriculum/internal/IndividualFiledSubjectDataExtractor.kt
@@ -13,9 +13,16 @@ class IndividualFiledSubjectDataExtractor(private val skolverketFileArchive: Sko
     override fun getSubjectData(): List<SubjectHtml> {
         return skolverketFileArchive.getFileStreams(schoolType.archivePath).map {
             getSubject(it)
-        }.toList().filter { filterBySpecSchoolType(it.code) }
+        }.toList().filter { filterBySpecSchoolType(it.code) } + getSubjectAreaData()
+
     }
 
+    private fun getSubjectAreaData(): List<SubjectHtml> {
+        if (schoolType.archivePathSubjectArea != null) {
+            return skolverketFileArchive.getFileStreams(schoolType.archivePathSubjectArea).map { getSubject(it, SyllabusType.SUBJECT_AREA_SYLLABUS) }.toList()
+        }
+        return emptyList()
+    }
     /**
      * Special school is a mix of special syllabuses together with the gr syllabuses
      * This parser splits these into different types
@@ -28,15 +35,19 @@ class IndividualFiledSubjectDataExtractor(private val skolverketFileArchive: Sko
         }
     }
 
-    private inline fun <reified T : kotlin.Enum<T>> valueOfOrNull(type: String?): T? {
+    private inline fun <reified T : Enum<T>> valueOfOrNull(type: String?): T? {
         return try {
-            java.lang.Enum.valueOf(T::class.java, type)
+            if (type != null) {
+                java.lang.Enum.valueOf(T::class.java, type)
+            } else {
+                null
+            }
         } catch (ia: IllegalArgumentException) {
             null
         }
     }
 
-    private fun getSubject(openDataDocumentStream: InputStream): SubjectHtml {
+    private fun getSubject(openDataDocumentStream: InputStream, typeOfSyllabus: SyllabusType? = null): SubjectHtml {
         val openDataDocument = Jsoup.parse(openDataDocumentStream, null, "", Parser.xmlParser())
         fun extractString(elementName: String): String = openDataDocument.select("subject > $elementName" ).text()
 
@@ -48,10 +59,10 @@ class IndividualFiledSubjectDataExtractor(private val skolverketFileArchive: Sko
                 extractString("designation"),
                 extractString("skolfsId"),
                 convertDashListToList(extractString("purpose")),
-                extractCourses(openDataDocument),
+                extractCourses(openDataDocument, typeOfSyllabus),
                 extractString("createdDate"),
                 extractString("modifiedDate"),
-                valueOfOrNull<SyllabusType>(extractString("typeOfSyllabus")),
+                valueOfOrNull<SyllabusType>(extractString("typeOfSyllabus")) ?: typeOfSyllabus,
                 valueOfOrNull<TypeOfSchooling>(extractString("typeOfSchooling")),
                 valueOfOrNull<TypeOfSchooling>(extractString("originatorTypeOfSchooling")),
                 extractString("gradeScale"),
@@ -60,17 +71,17 @@ class IndividualFiledSubjectDataExtractor(private val skolverketFileArchive: Sko
            )
     }
 
-    private fun extractCourses(openDataDocument: Document): List<CourseHtml> {
+    private fun extractCourses(openDataDocument: Document, typeOfSyllabus: SyllabusType? = null): List<CourseHtml> {
+        if (typeOfSyllabus == SyllabusType.SUBJECT_AREA_SYLLABUS) {
+            return SubjectAreaDataExtractor(openDataDocument).getCourseData()
+        }
         // Get the list of courses and return as CoursePOJOs
         return when (schoolType) {
             SchoolType.GY, SchoolType.GYS ->
                 UpperSecondaryCourseDataExtractor(openDataDocument).getCourseData()
-            SchoolType.GYS_SUBJECT_AREA ->
-                SubjectAreaDataExtractor(openDataDocument).getCourseData()
-            SchoolType.SFI -> SFICourseDataExtractor(openDataDocument).getCourseData()
-            SchoolType.VUXGR ->
-                VuxCourseDataExtractor(openDataDocument).getCourseData()
-            SchoolType.VUXGRS ->
+            SchoolType.SFI ->
+                SFICourseDataExtractor(openDataDocument).getCourseData()
+            SchoolType.VUXGRS, SchoolType.VUXGR ->
                 VuxCourseDataExtractor(openDataDocument).getCourseData()
             SchoolType.GR, SchoolType.GRS, SchoolType.GRSPEC, SchoolType.SPEC, SchoolType.GRSSPEC, SchoolType.GRSAM ->
                 CompulsoryCourseDataExtractor(openDataDocument).getCourseData()

diff --git a/src/main/kotlin/org/edtech/curriculum/internal/SubjectAreaDataExtractor.kt b/src/main/kotlin/org/edtech/curriculum/internal/SubjectAreaDataExtractor.kt
@@ -24,7 +24,7 @@ class SubjectAreaDataExtractor(private val subjectDocument: Document): CourseDat
                 subjectDocument.select("code").text(),
                 "",
                 "",
-               "",
+                subjectDocument.select("point").text(),
                 convertDashListToList(subjectDocument.select("centralContents").text()),
                 this.getKnowledgeRequirements(subjectDocument.select("knowledgeRequirements"))))
     }

diff --git a/src/main/kotlin/org/edtech/curriculum/internal/VuxSubjectAreaDataExtractor.kt b/src/main/kotlin/org/edtech/curriculum/internal/VuxSubjectAreaDataExtractor.kt
@@ -0,0 +1,42 @@
+package org.edtech.curriculum.internal
+
+import org.edtech.curriculum.CourseHtml
+import org.edtech.curriculum.GradeStep
+import org.edtech.curriculum.RequirementGroup
+import org.jsoup.nodes.Document
+import org.jsoup.select.Elements
+
+
+/**
+ * Parses the course data in the format supplied in syllabus.tgz
+ *
+ * Extracts the information to be used for processing
+ *
+ *
+ * @param subjectDocument to extract information from
+ */
+class VuxSubjectAreaDataExtractor(private val subjectDocument: Document): CourseDataExtractor {
+
+    override fun getCourseData(): List<CourseHtml> {
+        return listOf(CourseHtml(
+                subjectDocument.select("name").text(),
+                subjectDocument.select("description").text().removePrefix("<p>").removeSuffix("</p>"),
+                subjectDocument.select("code").text(),
+                subjectDocument.select("category").text(),
+                "",
+                subjectDocument.select("point").text(),
+                convertDashListToList(subjectDocument.select("centralContents text").text()),
+                this.getKnowledgeRequirements(subjectDocument.select("knowledgeRequirements"))))
+    }
+
+
+    private fun getKnowledgeRequirements(knowledgeRequirementElements: Elements): List<RequirementGroup> {
+        return listOf(RequirementGroup(knowledgeRequirementElements.map {
+            Pair(
+                    GradeStep.valueOf(it.selectFirst("typeOfRequirement").text()),
+                    it.selectFirst("text").text()
+            )
+        }.toMap()))
+    }
+}
+
diff --git a/src/test/kotlin/org/edtech/curriculum/CurriculumTest.kt b/src/test/kotlin/org/edtech/curriculum/CurriculumTest.kt
@@ -57,7 +57,7 @@ class CurriculumTest {
 
                         // Only check real courses
                         if (schoolType != SchoolType.SFI) {
-                            if (courseHtml.year.isEmpty() && schoolType != SchoolType.GYS_SUBJECT_AREA) {
+                            if (courseHtml.year.isEmpty() && subjectHtml.typeOfSyllabus != SyllabusType.SUBJECT_AREA_SYLLABUS) {
                                 if (courseHtml.point.isEmpty()) {
                                     fail<Unit>("${courseHtml.code}/${courseHtml.name} has no points/year group")
                                 }
@@ -158,7 +158,7 @@ class CurriculumTest {
                             assertTrue(course.name.isNotEmpty(), "${course.code}/${course.name} has no name")
                             // Only check real courses
                             if (schoolType != SchoolType.SFI) {
-                                if (course.year == null && schoolType != SchoolType.GYS_SUBJECT_AREA) {
+                                if (course.year == null && subject.typeOfSyllabus != SyllabusType.SUBJECT_AREA_SYLLABUS) {
                                     if (course.point == null) {
                                         fail<Unit>("${course.code}/${course.name} has no points/year group")
                                     }

diff --git a/src/test/kotlin/org/edtech/curriculum/SkolverketFileArchiveTest.kt b/src/test/kotlin/org/edtech/curriculum/SkolverketFileArchiveTest.kt
@@ -20,20 +20,40 @@ class SkolverketFileArchiveTest {
         }
     }
 
+
+    @TestFactory
+    fun testSkolverketArchivesSubjectArea() = SchoolType.values()
+            .filter { schoolType -> schoolType.archivePathSubjectArea != null }
+            .map { schoolType ->
+        DynamicTest.dynamicTest(schoolType.name + "Subject Area") {
+            val sf = SkolverketFileArchive(File(dataDir, schoolType.filename))
+            assertTrue(sf.archiveExists(), "${schoolType.filename} does not exist")
+            assertTrue(sf.fileExists(schoolType.archivePathSubjectArea!!), "${schoolType.archivePathSubjectArea} does not exist in ${schoolType.filename}")
+            assertEquals(expectedNumberOfXMLFilesSubjectArea(schoolType),
+                    sf.getFileStreams(schoolType.archivePathSubjectArea!!).size, "${schoolType.filename } -> ${schoolType.name} / ${schoolType.archivePathSubjectArea} does not contain expected number of xml-files")
+        }
+    }
+
     private fun expectedNumberOfXMLFiles(schoolType: SchoolType): Int {
+            return when (schoolType) {
+                SchoolType.GR -> 25
+                SchoolType.GRS -> 23
+                SchoolType.GRSAM -> 25
+                SchoolType.GRSPEC -> 34
+                SchoolType.GY -> 294
+                SchoolType.VUXGR -> 15
+                SchoolType.VUXGRS -> 13
+                SchoolType.GYS -> 74
+                SchoolType.SFI -> 1
+                SchoolType.SPEC -> 34
+                SchoolType.GRSSPEC -> 23
+            }
+    }
+    private fun expectedNumberOfXMLFilesSubjectArea(schoolType: SchoolType): Int {
         return when (schoolType) {
-            SchoolType.GR -> 25
-            SchoolType.GRS -> 23
-            SchoolType.GRSAM -> 25
-            SchoolType.GRSPEC -> 34
-            SchoolType.GY -> 294
-            SchoolType.VUXGR -> 15
-            SchoolType.VUXGRS -> 13
-            SchoolType.GYS -> 74
-            SchoolType.GYS_SUBJECT_AREA -> 6
-            SchoolType.SFI -> 1
-            SchoolType.SPEC -> 34
-            SchoolType.GRSSPEC -> 23
+            SchoolType.GYS -> 6
+            SchoolType.VUXGRS -> 3
+            else -> 0
         }
     }
 }