[#11240] [#11238] Support Docker compose for dependent service + depl…

…oyment to GAE flex (#11253)

.github/workflows/component.yml

@@ -64,10 +64,7 @@ jobs:
         run: mv src/test/resources/test.ci-${{ matrix.os }}.properties src/test/resources/test.properties
       - name: Run Solr search service
         if: matrix.os == 'ubuntu-latest' # Docker does not work well on Windows env
-        run: |
-          docker run --name=tm_solr -d -p 8983:8983 solr:8.8.1
-          sleep 5
-          docker exec $(docker ps -qf "name=tm_solr") /bin/sh -c "$(cat ./solr.sh)"
+        run: docker-compose run -d -p 8983:8983 solr
       - name: Run Backend Tests
         run: |
           ./gradlew createConfigs componentTests 

.github/workflows/e2e.yml

@@ -37,20 +37,17 @@ jobs:
         run: mv $GITHUB_WORKSPACE/src/e2e/resources/test.ci-$E2E_BROWSER.properties src/e2e/resources/test.properties
         env:
           E2E_BROWSER: ${{ matrix.browser }}
-      - name: Run Solr search service
-        run: |
-          docker run --name=tm_solr -d -p 8983:8983 solr:8.8.1
-          sleep 5
-          docker exec $(docker ps -qf "name=tm_solr") /bin/sh -c "$(cat ./solr.sh)"
+      - name: Run Solr search service + local Datastore emulator
+        run: docker-compose up -d
       - name: Create Config Files
         run: ./gradlew createConfigs testClasses generateTypes 
-      - name: Run local Datastore emulator
-        run: ./gradlew runDatastoreEmulator
       - name: Install Frontend Dependencies
         run: npm ci 
       - name: Build Frontend Bundle 
         run: npm run build -- --progress=false --serviceWorker=false 
       - name: Start Server
         run: ./gradlew serverRun & 
+      - name: Wait until server is running
+        run: ./wait-for-server.sh
       - name: Start Tests 
         run: xvfb-run --server-args="-screen 0 1024x768x24" ./gradlew e2eTests 

.github/workflows/lnp.yml

@@ -25,9 +25,11 @@ jobs:
             ${{ runner.os }}-gradle-
       - name: Create Config Files
         run: ./gradlew createConfigs
-      - name: Run local Datastore emulator
-        run: ./gradlew runDatastoreEmulator
+      - name: Run Solr search service + local Datastore emulator
+        run: docker-compose up -d
       - name: Start Server
         run: ./gradlew serverRun &
+      - name: Wait until server is running
+        run: ./wait-for-server.sh
       - name: Start Tests
         run: ./gradlew lnpTests

.gitignore

@@ -71,6 +71,6 @@ src/e2e/resources/gmail-api/
 src/e2e/resources/downloads/
 src/web/dist/*
 filestorage-dev/*
-datastore-dev/*
+datastore-dev/datastore/*
 
 !.gitkeep

build.gradle

@@ -314,8 +314,13 @@ task copyDistToStagedApp {
 }
 
 appengineStage {
-    dependsOn explodeWar
-    finalizedBy removeWarFromStagedApp, copyExplodedWarToStagedApp, copyDistToStagedApp
+    if (project.hasProperty("flex")) {
+        dependsOn explodeWar
+        finalizedBy removeWarFromStagedApp, copyExplodedWarToStagedApp, copyDistToStagedApp
+    } else {
+        dependsOn explodeWar, copyDistToExplodedWar
+        finalizedBy removeWarFromStagedApp, copyExplodedWarToStagedApp
+    }
 }
 
 // STATIC ANALYSIS TASKS

datastore-dev/Dockerfile

@@ -0,0 +1,6 @@
+FROM gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+
+RUN apk --update add openjdk8-jre
+RUN gcloud components install beta cloud-datastore-emulator
+
+ENTRYPOINT gcloud beta emulators datastore start --host-port 0.0.0.0:8484 --project placeholder

docker-compose.yml

@@ -0,0 +1,12 @@
+version: "3.9"
+services:
+  datastore:
+    build:
+      context: datastore-dev
+    ports:
+      - 8484:8484
+  solr:
+    build:
+      context: solr
+    ports:
+      - 8983:8983

docs/development.md

@@ -57,6 +57,11 @@ In order for the back-end to properly work, you need to have a running database
 
 The details on how to run them locally can be found [here (for local Datastore emulator)](#running-the-datastore-emulator) and [here (for full-text search service)](search.md).
 
+If you have access to Docker, we have a Docker compose definition to run those services:
+```sh
+docker-compose up -d
+```
+
 ### Starting the dev server
 
 To start the server in the background, run the following command
@@ -171,6 +176,13 @@ You can use the pre-provided quickstart script which will run a local Datastore
 
 The Datastore emulator will be running in the port specified in the `build.properties` file.
 
+### Using Docker-based tooling
+
+We have a Docker compose definition to run dependent services, including local Datastore emulator. Run it under the `datastore` service name and bind to the container port `8484`:
+```sh
+docker-compose run -p 8484:8484 datastore
+```
+
 ### Using Cloud SDK
 
 Alternatively, you can use `gcloud` command to manage the local Datastore emulator instance directly. For this, you need a working [Google Cloud SDK](https://cloud.google.com/sdk/docs) in your development environment.

docs/search.md

@@ -8,18 +8,14 @@ This document will assume Solr version `8.8.1`.
 
 ## Setting up Solr using Docker
 
-If you are familiar with Docker, this method is recommended.
+If you have access to Docker, this method is straightforward and recommended.
 
-1. Run the `solr:8.8.1` Docker image and bind to the container port `8983`. For example, to run a container named `tm_solr` accessible from `localhost:8983` in the background:
-   ```sh
-   docker pull solr:8.8.1
-   docker run --name=tm_solr -d -p 8983:8983 solr:8.8.1
-   ```
-   **Verification:** the Solr admin console should be accessible in `http://localhost:8983`.
-1. To initialise Solr for our use cases, we run the [Solr startup script](../solr.sh) located in the project directory in the running container:
-   ```sh
-   docker exec $(docker ps -qf "name=tm_solr") /bin/sh -c "$(cat ./solr.sh)"
-   ```
+We have provided a Docker compose definition to run dependent services, including Solr. Run it under the `solr` service name and bind to the container port `8983`:
+```sh
+docker-compose run -p 8983:8983 solr
+```
+
+**Verification:** the Solr admin console should be accessible in `http://localhost:8983`.
 
 ## Setting up Solr manually
 

solr/Dockerfile

@@ -0,0 +1,5 @@
+FROM solr:8.8.1
+
+COPY solr.sh solr.sh
+
+ENTRYPOINT ["/bin/sh", "-c", "bin/solr start && ./solr.sh && tail -f /var/solr/logs/solr.log"]

src/e2e/java/teammates/e2e/cases/AdminAccountsPageE2ETest.java

@@ -64,6 +64,7 @@ public void testAll() {
 
         accountsPage.clickDowngradeAccount();
         accountsPage.verifyStatusMessage("Instructor account is successfully downgraded to student.");
+        accountsPage.waitForPageToLoad();
 
         account = getAccount(googleId);
         assertFalse(account.isInstructor);

src/e2e/java/teammates/e2e/cases/AdminSessionsPageE2ETest.java

@@ -74,7 +74,6 @@ public void testAll() {
 
         AppUrl sessionsUrl = createUrl(Const.WebPageURIs.ADMIN_SESSIONS_PAGE);
         AdminSessionsPage sessionsPage = loginAdminToPage(sessionsUrl, AdminSessionsPage.class);
-        sessionsPage.waitUntilAnimationFinish();
 
         String tableTimezone = sessionsPage.getSessionsTableTimezone();
 

src/e2e/java/teammates/e2e/cases/BaseE2ETestCase.java

@@ -154,7 +154,7 @@ protected void deleteDownloadsFile(String fileName) {
      */
     protected void verifyDownloadedFile(String expectedFileName, List<String> expectedContent) {
         String filePath = getTestDownloadsFolder() + expectedFileName;
-        int retryLimit = 5;
+        int retryLimit = TestProperties.TEST_TIMEOUT;
         boolean actual = Files.exists(Paths.get(filePath));
         while (!actual && retryLimit > 0) {
             retryLimit--;
@@ -174,7 +174,8 @@ protected void verifyDownloadedFile(String expectedFileName, List<String> expect
     }
 
     protected <T extends AppPage> T getNewPageInstance(AppUrl url, Class<T> typeOfPage) {
-        return AppPage.getNewPageInstance(browser, url, typeOfPage);
+        browser.goToUrl(url.toAbsoluteString());
+        return AppPage.getNewPageInstance(browser, typeOfPage);
     }
 
     /**

src/e2e/java/teammates/e2e/cases/FeedbackConstSumOptionQuestionE2ETest.java

@@ -78,6 +78,7 @@ protected void testEditPage() {
         questionDetails.setDistributePointsFor("At least some options");
         loadedQuestion.questionDetails = questionDetails;
         feedbackEditPage.editConstSumQuestion(2, questionDetails);
+        feedbackEditPage.waitForPageToLoad();
 
         feedbackEditPage.verifyConstSumQuestionDetails(2, questionDetails);
         verifyPresentInDatastore(loadedQuestion);

src/e2e/java/teammates/e2e/cases/FeedbackConstSumRecipientQuestionE2ETest.java

@@ -75,6 +75,7 @@ protected void testEditPage() {
         questionDetails.setDistributePointsFor("At least some options");
         loadedQuestion.questionDetails = questionDetails;
         feedbackEditPage.editConstSumQuestion(2, questionDetails);
+        feedbackEditPage.waitForPageToLoad();
 
         feedbackEditPage.verifyConstSumQuestionDetails(2, questionDetails);
         verifyPresentInDatastore(loadedQuestion);

src/e2e/java/teammates/e2e/cases/FeedbackContributionQuestionE2ETest.java

@@ -74,6 +74,7 @@ protected void testEditPage() {
         questionDetails.setNotSureAllowed(false);
         loadedQuestion.questionDetails = questionDetails;
         feedbackEditPage.editContributionQuestion(2, questionDetails);
+        feedbackEditPage.waitForPageToLoad();
 
         feedbackEditPage.verifyContributionQuestionDetails(2, questionDetails);
         verifyPresentInDatastore(loadedQuestion);

src/e2e/java/teammates/e2e/cases/FeedbackMcqQuestionE2ETest.java

@@ -79,6 +79,7 @@ protected void testEditPage() {
         questionDetails.setMcqChoices(choices);
         loadedQuestion.questionDetails = questionDetails;
         feedbackEditPage.editMcqQuestion(2, questionDetails);
+        feedbackEditPage.waitForPageToLoad();
 
         feedbackEditPage.verifyMcqQuestionDetails(2, questionDetails);
         verifyPresentInDatastore(loadedQuestion);

src/e2e/java/teammates/e2e/cases/FeedbackMsqQuestionE2ETest.java

@@ -82,6 +82,7 @@ protected void testEditPage() {
         questionDetails.setMsqChoices(choices);
         loadedQuestion.questionDetails = questionDetails;
         feedbackEditPage.editMsqQuestion(2, questionDetails);
+        feedbackEditPage.waitForPageToLoad();
 
         feedbackEditPage.verifyMsqQuestionDetails(2, questionDetails);
         verifyPresentInDatastore(loadedQuestion);

src/e2e/java/teammates/e2e/cases/FeedbackNumScaleQuestionE2ETest.java

@@ -49,6 +49,7 @@ protected void testEditPage() {
         // add new question exactly like loaded question
         loadedQuestion.setQuestionNumber(2);
         feedbackEditPage.addNumScaleQuestion(loadedQuestion);
+        feedbackEditPage.waitUntilAnimationFinish();
 
         feedbackEditPage.verifyNumScaleQuestionDetails(2, questionDetails);
         verifyPresentInDatastore(loadedQuestion);
@@ -72,6 +73,7 @@ protected void testEditPage() {
         questionDetails.setMaxScale(100);
         loadedQuestion.questionDetails = questionDetails;
         feedbackEditPage.editNumScaleQuestion(2, questionDetails);
+        feedbackEditPage.waitForPageToLoad();
 
         feedbackEditPage.verifyNumScaleQuestionDetails(2, questionDetails);
         verifyPresentInDatastore(loadedQuestion);

src/e2e/java/teammates/e2e/cases/FeedbackRankOptionQuestionE2ETest.java

@@ -81,6 +81,7 @@ protected void testEditPage() {
         questionDetails.setMinOptionsToBeRanked(1);
         loadedQuestion.questionDetails = questionDetails;
         feedbackEditPage.editRankQuestion(2, questionDetails);
+        feedbackEditPage.waitForPageToLoad();
 
         feedbackEditPage.verifyRankQuestionDetails(2, questionDetails);
         verifyPresentInDatastore(loadedQuestion);

src/e2e/java/teammates/e2e/cases/FeedbackRankRecipientQuestionE2ETest.java

@@ -77,6 +77,7 @@ protected void testEditPage() {
         questionDetails.setMinOptionsToBeRanked(Const.POINTS_NO_VALUE);
         loadedQuestion.questionDetails = questionDetails;
         feedbackEditPage.editRankQuestion(2, questionDetails);
+        feedbackEditPage.waitForPageToLoad();
 
         feedbackEditPage.verifyRankQuestionDetails(2, questionDetails);
         verifyPresentInDatastore(loadedQuestion);

src/e2e/java/teammates/e2e/cases/FeedbackRubricQuestionE2ETest.java

@@ -93,6 +93,7 @@ protected void testEditPage() {
         questionDetails.setRubricWeightsForEachCell(new ArrayList<>());
         loadedQuestion.questionDetails = questionDetails;
         feedbackEditPage.editRubricQuestion(2, questionDetails);
+        feedbackEditPage.waitForPageToLoad();
 
         feedbackEditPage.verifyRubricQuestionDetails(2, questionDetails);
         verifyPresentInDatastore(loadedQuestion);

src/e2e/java/teammates/e2e/cases/InstructorCoursesPageE2ETest.java

@@ -110,6 +110,7 @@ public void testAll() {
         coursesPage.restoreCourse(newCourse.getId());
 
         coursesPage.verifyStatusMessage("The course " + newCourse.getId() + " has been restored.");
+        coursesPage.waitForPageToLoad();
         coursesPage.verifyNumDeletedCourses(1);
         // No need to call sortByCreationDate() here because it is the default sort in DESC order
         coursesPage.verifyActiveCoursesDetails(activeCoursesWithNewCourseSortedByCreationDate);
@@ -131,6 +132,7 @@ public void testAll() {
         coursesPage.restoreCourse(newCourse.getId());
 
         coursesPage.verifyStatusMessage("The course " + newCourse.getId() + " has been restored.");
+        coursesPage.waitForPageToLoad();
         coursesPage.verifyNumDeletedCourses(1);
         coursesPage.verifyArchivedCoursesDetails(archivedCoursesWithNewCourse);
         assertFalse(BACKDOOR.isCourseInRecycleBin(newCourse.getId()));
@@ -151,6 +153,7 @@ public void testAll() {
         coursesPage.restoreAllCourses();
 
         coursesPage.verifyStatusMessage("All courses have been restored.");
+        coursesPage.waitForPageToLoad();
         coursesPage.sortByCourseId();
         coursesPage.verifyActiveCoursesDetails(activeCoursesWithRestored);
         coursesPage.verifyArchivedCoursesDetails(archivedCourses);

src/e2e/java/teammates/e2e/cases/TimezoneSyncerTest.java

@@ -52,7 +52,6 @@ public void testAll() {
         String currentTzVersion = timezonePage.getMomentTimezoneVersion();
         IanaTimezonePage ianaPage = getNewPageInstance(
                 new AppUrl(IanaTimezonePage.IANA_TIMEZONE_DATABASE_URL), IanaTimezonePage.class);
-        ianaPage.waitForPageToLoad();
         String latestTzVersion = ianaPage.getVersion();
 
         if (!currentTzVersion.equals(latestTzVersion)) {

src/e2e/java/teammates/e2e/pageobjects/AppPage.java

@@ -9,6 +9,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 import java.util.Map;
 
@@ -29,7 +30,6 @@
 
 import teammates.common.datatransfer.FeedbackParticipantType;
 import teammates.common.util.ThreadHelper;
-import teammates.common.util.Url;
 import teammates.common.util.retry.MaximumRetriesExceededException;
 import teammates.common.util.retry.RetryManager;
 import teammates.common.util.retry.RetryableTask;
@@ -49,12 +49,16 @@ public abstract class AppPage {
 
     private static final String CLEAR_ELEMENT_SCRIPT;
     private static final String SCROLL_ELEMENT_TO_CENTER_AND_CLICK_SCRIPT;
+    private static final String READ_TINYMCE_CONTENT_SCRIPT;
+    private static final String WRITE_TO_TINYMCE_SCRIPT;
 
     static {
         try {
             CLEAR_ELEMENT_SCRIPT = FileHelper.readFile("src/e2e/resources/scripts/clearElementWithoutEvents.js");
             SCROLL_ELEMENT_TO_CENTER_AND_CLICK_SCRIPT = FileHelper
                     .readFile("src/e2e/resources/scripts/scrollElementToCenterAndClick.js");
+            READ_TINYMCE_CONTENT_SCRIPT = FileHelper.readFile("src/e2e/resources/scripts/readTinyMCEContent.js");
+            WRITE_TO_TINYMCE_SCRIPT = FileHelper.readFile("src/e2e/resources/scripts/writeToTinyMCE.js");
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
@@ -104,27 +108,24 @@ public AppPage(Browser browser) {
         throw new IllegalStateException("Not in the correct page!");
     }
 
-    /**
-     * Fails if the new page content does not match content expected in a page of
-     * the type indicated by the parameter {@code typeOfPage}.
-     */
-    public static <T extends AppPage> T getNewPageInstance(Browser currentBrowser, Url url, Class<T> typeOfPage) {
-        currentBrowser.goToUrl(url.toAbsoluteString());
-        waitUntilAnimationFinish(currentBrowser);
-        return getNewPageInstance(currentBrowser, typeOfPage);
-    }
-
     /**
      * Fails if the new page content does not match content expected in a page of
      * the type indicated by the parameter {@code typeOfPage}.
      */
     public static <T extends AppPage> T getNewPageInstance(Browser currentBrowser, Class<T> typeOfPage) {
+        waitUntilAnimationFinish(currentBrowser);
         try {
             Constructor<T> constructor = typeOfPage.getConstructor(Browser.class);
             T page = constructor.newInstance(currentBrowser);
             PageFactory.initElements(currentBrowser.driver, page);
+            page.waitForPageToLoad();
             return page;
-        } catch (Exception e) {
+        } catch (InvocationTargetException e) {
+            if (e.getCause() instanceof IllegalStateException) {
+                throw (IllegalStateException) e.getCause();
+            }
+            throw new RuntimeException(e);
+        } catch (NoSuchMethodException | IllegalAccessException | InstantiationException e) {
             throw new RuntimeException(e);
         }
     }
@@ -347,13 +348,9 @@ protected void fillFileBox(RemoteWebElement fileBoxElement, String fileName) {
      */
     protected String getEditorRichText(WebElement editor) {
         waitForElementPresence(By.tagName("iframe"));
-        browser.driver.switchTo().frame(editor.findElement(By.tagName("iframe")));
-
-        String innerHtml = browser.driver.findElement(By.id("tinymce")).getAttribute("innerHTML");
-        // check if editor is empty
-        innerHtml = innerHtml.contains("data-mce-bogus") ? "" : innerHtml;
-        browser.driver.switchTo().defaultContent();
-        return innerHtml;
+        String id = editor.findElement(By.tagName("textarea")).getAttribute("id");
+        return (String) ((JavascriptExecutor) browser.driver)
+                .executeAsyncScript(READ_TINYMCE_CONTENT_SCRIPT, id);
     }
 
     /**
@@ -362,8 +359,7 @@ protected String getEditorRichText(WebElement editor) {
     protected void writeToRichTextEditor(WebElement editor, String text) {
         waitForElementPresence(By.tagName("iframe"));
         String id = editor.findElement(By.tagName("textarea")).getAttribute("id");
-        executeScript(String.format("tinyMCE.get('%s').setContent('%s');"
-                + " tinyMCE.get('%s').save()", id, text, id));
+        ((JavascriptExecutor) browser.driver).executeAsyncScript(WRITE_TO_TINYMCE_SCRIPT, id, text);
     }
 
     /**