Java: Add tests for XML and Java extraction with GBK charset

java/ql/integration-tests/java/maven-sample-xml-mode-all-gbk-encoding/pom.xml

@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>com.example</groupId>
+  <artifactId>maven-sample</artifactId>
+  <version>1.0-SNAPSHOT</version>
+
+  <name>maven-sample</name>
+  <!-- FIXME change it to the project's website -->
+  <url>http://www.example.com</url>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.source>1.7</maven.compiler.source>
+    <maven.compiler.target>1.7</maven.compiler.target>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.11</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>exec-maven-plugin</artifactId>
+        <groupId>org.codehaus.mojo</groupId>
+        <version>1.1.1</version>
+        <executions>
+          <execution>
+            <id>check-maven-version</id>
+            <phase>package</phase>
+            <goals>
+              <goal>java</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <mainClass>com.example.App</mainClass>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>com.diffplug.spotless</groupId>
+        <artifactId>spotless-maven-plugin</artifactId>
+        <version>2.19.1</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>check</goal>
+            </goals>
+            <phase>compile</phase>
+          </execution>
+        </executions>
+        <configuration>
+          <java>
+            <licenseHeader>
+              <content>/* FAIL ME */</content>
+            </licenseHeader>
+          </java>
+        </configuration>
+      </plugin>
+    </plugins>
+    <pluginManagement>
+      <!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
+      <plugins>
+        <!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
+        <plugin>
+          <artifactId>maven-clean-plugin</artifactId>
+          <version>3.1.0</version>
+        </plugin>
+        <!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
+        <plugin>
+          <artifactId>maven-resources-plugin</artifactId>
+          <version>3.0.2</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-compiler-plugin</artifactId>
+          <version>3.8.0</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-surefire-plugin</artifactId>
+          <version>2.22.1</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-jar-plugin</artifactId>
+          <version>3.0.2</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-install-plugin</artifactId>
+          <version>2.5.2</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-deploy-plugin</artifactId>
+          <version>2.8.2</version>
+        </plugin>
+        <!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
+        <plugin>
+          <artifactId>maven-site-plugin</artifactId>
+          <version>3.7.1</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-project-info-reports-plugin</artifactId>
+          <version>3.0.0</version>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+</project>

java/ql/integration-tests/java/maven-sample-xml-mode-all-gbk-encoding/source_archive.expected

@@ -0,0 +1,8 @@
+pom.xml
+src/main/java/com/example/App.java
+src/main/resources/my-app.properties
+src/main/resources/page.xml
+src/test/java/com/example/AppTest.java
+target/classes/my-app.properties
+target/classes/page.xml
+target/maven-archiver/pom.properties

java/ql/integration-tests/java/maven-sample-xml-mode-all-gbk-encoding/src/main/java/com/example/App.java

@@ -0,0 +1,30 @@
+package com.example;
+
+import java.util.regex.Pattern;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * Hello world!
+ *
+ */
+public class App 
+{
+    public static void main( String[] args )
+    {
+        System.out.println( "Hello World!" );
+        String expectedVersion = System.getenv("EXPECT_MAVEN");
+        Path mavenHome = Paths.get(System.getProperty("maven.home")).normalize();
+        String observedVersion = mavenHome.getFileName().toString();
+        if (expectedVersion != null && !expectedVersion.equals(observedVersion)) {
+                System.err.println("Wrong maven version, expected '" + expectedVersion + "' but got '" + observedVersion + "'" + mavenHome);
+                System.exit(1);
+        }
+        String commandMatcher = System.getenv("EXPECT_COMMAND_REGEX");
+        String command = System.getProperty("sun.java.command");
+        if (commandMatcher != null && !Pattern.matches(commandMatcher, command)) {
+                System.err.println("Wrong command line, '" + command + "' does not match '" + commandMatcher + "'");
+                System.exit(1);
+        }
+    }
+}

java/ql/integration-tests/java/maven-sample-xml-mode-all-gbk-encoding/src/main/resources/my-app.properties

@@ -0,0 +1 @@
+version=1.0

java/ql/integration-tests/java/maven-sample-xml-mode-all-gbk-encoding/src/main/resources/page.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="GBK" ?>
+<doc>ÄãºÃÊÀ½ç</doc>

java/ql/integration-tests/java/maven-sample-xml-mode-all-gbk-encoding/src/test/java/com/example/AppTest.java

@@ -0,0 +1,20 @@
+package com.example;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+/**
+ * Unit test for simple App.
+ */
+public class AppTest 
+{
+    /**
+     * Rigorous Test :-)
+     */
+    @Test
+    public void shouldAnswerWithTrue()
+    {
+        assertTrue( true );
+    }
+}

java/ql/integration-tests/java/maven-sample-xml-mode-all-gbk-encoding/test.expected

@@ -0,0 +1,2 @@
+| src/main/resources/page.xml:2:6:2:10 | \u4f60\u597d\u4e16\u754c |
+| target/classes/page.xml:2:6:2:10 | \u4f60\u597d\u4e16\u754c |

java/ql/integration-tests/java/maven-sample-xml-mode-all-gbk-encoding/test.py

@@ -0,0 +1,2 @@
+def test(codeql, java):
+    codeql.database.create(_env={"LGTM_INDEX_XML_MODE": "all"})

java/ql/integration-tests/java/maven-sample-xml-mode-all-gbk-encoding/test.ql

@@ -0,0 +1,5 @@
+import java
+
+from XmlElement e
+where e.hasName("doc")
+select e.getACharactersSet()

java/ql/test/library-tests/comment-encoding/PrintAst.expected

@@ -0,0 +1,3 @@
+Test.java:
+#    0| [CompilationUnit] Test
+#    1|   1: [Class] Test

java/ql/test/library-tests/comment-encoding/PrintAst.qlref

@@ -0,0 +1 @@
+semmle/code/java/PrintAst.ql

java/ql/test/library-tests/comment-encoding/Test.java

@@ -0,0 +1,4 @@
+public class Test {
+}
+
+// ÄãºÃÊÀ½ç

java/ql/test/library-tests/comment-encoding/options

@@ -0,0 +1 @@
+//semmle-extractor-options: --encoding GBK

java/ql/test/library-tests/comment-encoding/toString.expected

@@ -0,0 +1 @@
+| Test.java:4:1:4:7 | // \u4f60\u597d\u4e16\u754c |

java/ql/test/library-tests/comment-encoding/toString.ql

@@ -0,0 +1,4 @@
+import java
+
+from Javadoc c
+select c