diff --git a/.env b/.env new file mode 100644 index 0000000000..0b99a88cb5 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +COURSES_DIRECTORY_ON_HOST=../ww-docker-data/courses +WEBWORK2_HTTP_PORT_ON_HOST=8080 +WEBWORK_DB_USER=webworkWrite +WEBWORK_DB_PASSWORD=passwordRWsetItBeforeFirstStartingTheDBcontainer diff --git a/Dockerfile b/Dockerfile index 0c4dfdde2f..a3da47fc17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,90 @@ -FROM ubuntu:16.04 +# Optional things to change/configure below: +# +# 1. Which branch of webwork2/ and pg/ to install. +# +# 2. Installing the OPL in the Docker image itself. +# (almost 850MB: 290+MB for the main OPL, 90+MB for Pending, 460+MB for Contrib) +# +# By default this is NOT done, and it will instead be installed in +# a named Docker storage volume when the container is first started. +# +# Note: For typical use, we recommend that the OPL be either mounted from +# a local directory on the source or from a separate named data volume. +# That approach precludes needing to download the OPL for each update to +# the Docker image, and allows it to be easily upgraded using git in its +# persistent location. +# +# 3. Some things should be handled by setting environment variables which +# take effect at container startup. They can usually be set in +# docker-compose.yml. +# +# SSL=1 +# will turn on SSL at startup +# ADD_LOCALES="locale1,locale2,locale3" +# will build these locales at startup +# PAPERSIZE=a4 +# will set the system papersize to A4 +# SYSTEM_TIMEZONE=zone/city +# will set the system timezone to zone/city +# Make sure to use a valid setting. +# "/usr/bin/timedatectl list-timezones" on Ubuntu will find valid values +# ADD_APT_PACKAGES="package1 package2 package3" +# will have these additional Ubuntu packages installed at startup. +# +# ================================================================== -ENV PG_BRANCH=master \ - WEBWORK_URL=/webwork2 \ +# Phase 1 - download some Git repos for later use: +# as suggested by Nelson Moller in https://gist.github.com/nmoller/81bd8e149e6aa2a7cf051e0bf248b2e2 + +FROM alpine/git AS base + +# build args specifying the branches for webwork2 and pg used to build the image + +# To use the master branches of webwork2 and pg +#ARG WEBWORK2_BRANCH=master +#ARG PG_BRANCH=master +# To use the 2.15 branches of webwork2 and pg from the "official" GitHub repositories +ARG WEBWORK2_GIT_URL=https://github.com/openwebwork/webwork2.git +ARG WEBWORK2_BRANCH=WeBWorK-2.15 +ARG PG_GIT_URL=https://github.com/openwebwork/pg.git +ARG PG_BRANCH=PG-2.15 + +# assign the build args to the ENV variables +ENV WEBWORK2_GIT_URL_ENV ${WEBWORK2_GIT_URL} +ENV WEBWORK2_BRANCH_ENV ${WEBWORK2_BRANCH} +ENV PG_GIT_URL_ENV ${PG_GIT_URL} +ENV PG_BRANCH_ENV ${PG_BRANCH} + +WORKDIR /opt/base + +RUN echo Cloning branch $WEBWORK2_BRANCH_ENV from $WEBWORK2_GIT_URL_ENV \ + && echo git clone --single-branch --branch ${WEBWORK2_BRANCH_ENV} --depth 1 $WEBWORK2_GIT_URL_ENV \ + && git clone --single-branch --branch ${WEBWORK2_BRANCH_ENV} --depth 1 $WEBWORK2_GIT_URL_ENV \ + && rm -rf webwork2/.git webwork2/{*ignore,Dockerfile,docker-compose.yml,docker-config} + +RUN echo Cloning branch $PG_BRANCH_ENV branch from $PG_GIT_URL_ENV \ + && echo git clone --single-branch --branch ${PG_BRANCH_ENV} --depth 1 $PG_GIT_URL_ENV \ + && git clone --single-branch --branch ${PG_BRANCH_ENV} --depth 1 $PG_GIT_URL_ENV \ + && rm -rf pg/.git + +RUN git clone --single-branch --branch legacy-v2 --depth 1 https://github.com/mathjax/MathJax \ + && rm -rf MathJax/.git + +# Optional - include OPL (also need to uncomment further below when an included OPL is desired): +#RUN git clone --single-branch --branch master --depth 1 https://github.com/openwebwork/webwork-open-problem-library.git \ +# && rm -rf webwork-open-problem-library/.git + +# ================================================================== + +# Phase 2 - set ENV variables + +# we need to change FROM before setting the ENV variables + +FROM ubuntu:18.04 + +ENV WEBWORK_URL=/webwork2 \ WEBWORK_ROOT_URL=http://localhost \ - WEBWORK_DB_HOST=db \ - WEBWORK_DB_PORT=3306 \ - WEBWORK_DB_NAME=webwork \ - WEBWORK_DB_USER=webworkWrite \ - WEBWORK_DB_PASSWORD=passwordRW \ WEBWORK_SMTP_SERVER=localhost \ WEBWORK_SMTP_SENDER=webwork@example.com \ WEBWORK_TIMEZONE=America/New_York \ @@ -21,111 +97,199 @@ ENV PG_BRANCH=master \ # Only /var/log/apache2 is handled by /etc/logrotate.d/apache2. APACHE_LOG_DIR=/var/log/apache2 \ APP_ROOT=/opt/webwork \ + DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ DEV=0 -ENV WEBWORK_DB_DSN=DBI:mysql:${WEBWORK_DB_NAME}:${WEBWORK_DB_HOST}:${WEBWORK_DB_PORT} \ - WEBWORK_ROOT=$APP_ROOT/webwork2 \ +# Environment variables which depend on a prior environment variable must be set +# in an ENV call after the dependencies were defined. +ENV WEBWORK_ROOT=$APP_ROOT/webwork2 \ PG_ROOT=$APP_ROOT/pg \ PATH=$PATH:$APP_ROOT/webwork2/bin +# ================================================================== + +# Phase 3 - Ubuntu 18.04 base image + required packages + +# Packages changes/added for ubuntu 18.04: + +# For ubuntu 18.04 libemail-address-xs-perl installed from Ubuntu, for 16.04 it would be installed using cpamn +# +# texlive-generic-recommended # For ubuntu 16.04 - contains path.sty +# texlive-plain-generic # For ubuntu 18.04 - contains path.sty + +# Do NOT include "apt-get -y upgrade" +# see: https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ + RUN apt-get update \ && apt-get install -y --no-install-recommends --no-install-suggests \ - apache2 \ - curl \ - dvipng \ - gcc \ - libapache2-request-perl \ - libcrypt-ssleay-perl \ - libdatetime-perl \ - libdancer-perl \ - libdancer-plugin-database-perl \ - libdbd-mysql-perl \ - libemail-address-perl \ - libexception-class-perl \ - libextutils-xsbuilder-perl \ - libfile-find-rule-perl-perl \ - libgd-perl \ - libhtml-scrubber-perl \ - libjson-perl \ - liblocale-maketext-lexicon-perl \ - libmail-sender-perl \ - libmime-tools-perl \ - libnet-ip-perl \ - libnet-ldap-perl \ - libnet-oauth-perl \ - libossp-uuid-perl \ - libpadwalker-perl \ - libpath-class-perl \ - libphp-serialization-perl \ - libsoap-lite-perl \ - libsql-abstract-perl \ - libstring-shellquote-perl \ - libtemplate-perl \ - libtext-csv-perl \ - libtimedate-perl \ - libuuid-tiny-perl \ - libxml-parser-perl \ - libxml-writer-perl \ - libapache2-reload-perl \ - make \ - netpbm \ - preview-latex-style \ - texlive \ - texlive-latex-extra \ - libc6-dev \ - git \ - mysql-client \ - && curl -Lk https://cpanmin.us | perl - App::cpanminus \ - && cpanm install XML::Parser::EasyTree Iterator Iterator::Util Pod::WSDL Array::Utils HTML::Template XMLRPC::Lite Mail::Sender Email::Sender::Simple Data::Dump Statistics::R::IO \ - && rm -fr /var/lib/apt/lists/* ./cpanm /root/.cpanm /tmp/* - -RUN mkdir -p $APP_ROOT/courses $APP_ROOT/libraries $APP_ROOT/webwork2 - - -# Block to include webwork2 in the container, when needed, instead of getting it from a bind mount. -# Uncomment when needed, and set the correct branch name on the following line. -#ENV WEBWORK_BRANCH=master # need a valid branch name from https://github.com/openwebwork/webwork2 -#RUN curl -fSL https://github.com/openwebwork/webwork2/archive/${WEBWORK_BRANCH}.tar.gz -o /tmp/${WEBWORK_BRANCH}.tar.gz \ -# && cd /tmp \ -# && tar xzf /tmp/${WEBWORK_BRANCH}.tar.gz \ -# && mv webwork2-${WEBWORK_BRANCH} $APP_ROOT/webwork2 \ -# && rm -rf /tmp/${WEBWORK_BRANCH}.tar.gz /tmp/webwork2-${WEBWORK_BRANCH} - -RUN curl -fSL https://github.com/openwebwork/pg/archive/${PG_BRANCH}.tar.gz -o /tmp/${PG_BRANCH}.tar.gz \ - && tar xzf /tmp/${PG_BRANCH}.tar.gz \ - && mv pg-${PG_BRANCH} $APP_ROOT/pg \ - && rm /tmp/${PG_BRANCH}.tar.gz \ - && curl -fSL https://github.com/openwebwork/webwork-open-problem-library/archive/master.tar.gz -o /tmp/opl.tar.gz \ - && tar xzf /tmp/opl.tar.gz \ - && mv webwork-open-problem-library-master $APP_ROOT/libraries/webwork-open-problem-library \ - && rm /tmp/opl.tar.gz - -RUN curl -fSL https://github.com/mathjax/MathJax/archive/master.tar.gz -o /tmp/mathjax.tar.gz \ - && tar xzf /tmp/mathjax.tar.gz \ - && mv MathJax-master $APP_ROOT/MathJax \ - && rm /tmp/mathjax.tar.gz - #&& rm /tmp/VERSION - #curl -fSL https://github.com/openwebwork/webwork2/archive/WeBWorK-${WEBWORK_VERSION}.tar.gz -o /tmp/WeBWorK-${WEBWORK_VERSION}.tar.gz \ - #&& tar xzf /tmp/WeBWorK-${WEBWORK_VERSION}.tar.gz \ - #&& mv webwork2-WeBWorK-${WEBWORK_VERSION} $APP_ROOT/webwork2 \ - #&& rm /tmp/WeBWorK-${WEBWORK_VERSION}.tar.gz \ - - -RUN echo "PATH=$PATH:$APP_ROOT/webwork2/bin" >> /root/.bashrc - -COPY . $APP_ROOT/webwork2 - - -# Move these lines into docker-entrypoint.sh so the bind mount of courses -# will be available + apache2 \ + curl \ + dvipng \ + gcc \ + libapache2-request-perl \ + libcrypt-ssleay-perl \ + libdatetime-perl \ + libdancer-perl \ + libdancer-plugin-database-perl \ + libdbd-mysql-perl \ + libemail-address-xs-perl \ + libexception-class-perl \ + libextutils-xsbuilder-perl \ + libfile-find-rule-perl-perl \ + libgd-perl \ + libhtml-scrubber-perl \ + libjson-perl \ + liblocale-maketext-lexicon-perl \ + libmail-sender-perl \ + libmime-tools-perl \ + libnet-ip-perl \ + libnet-ldap-perl \ + libnet-oauth-perl \ + libossp-uuid-perl \ + libpadwalker-perl \ + libpath-class-perl \ + libphp-serialization-perl \ + libxml-simple-perl \ + libsoap-lite-perl \ + libsql-abstract-perl \ + libstring-shellquote-perl \ + libtemplate-perl \ + libtext-csv-perl \ + libtimedate-perl \ + libuuid-tiny-perl \ + libxml-parser-perl \ + libxml-writer-perl \ + libxmlrpc-lite-perl \ + libapache2-reload-perl \ + cpanminus \ + libxml-parser-easytree-perl \ + libiterator-perl \ + libiterator-util-perl \ + libpod-wsdl-perl \ + libtest-xml-perl \ + libmodule-build-perl \ + libxml-semanticdiff-perl \ + libxml-xpath-perl \ + libpath-tiny-perl \ + libarray-utils-perl \ + libhtml-template-perl \ + libtest-pod-perl \ + libemail-sender-perl \ + libmail-sender-perl \ + libmodule-pluggable-perl \ + libemail-date-format-perl \ + libcapture-tiny-perl \ + libthrowable-perl \ + libdata-dump-perl \ + libfile-sharedir-install-perl \ + libclass-tiny-perl \ + libtest-requires-perl \ + libtest-mockobject-perl \ + libtest-warn-perl \ + libsub-uplevel-perl \ + libtest-exception-perl \ + libuniversal-can-perl \ + libuniversal-isa-perl \ + libtest-fatal-perl \ + libjson-xs-perl \ + libmoox-options-perl \ + make \ + netpbm \ + preview-latex-style \ + texlive \ + texlive-latex-extra \ + texlive-plain-generic \ + texlive-xetex \ + texlive-latex-recommended \ + texlive-lang-other \ + texlive-lang-arabic \ + libc6-dev \ + git \ + mysql-client \ + tzdata \ + apt-utils \ + locales \ + debconf-utils \ + ssl-cert \ + ca-certificates \ + culmus \ + fonts-linuxlibertine \ + lmodern \ + zip \ + jq \ + && apt-get clean \ + && rm -fr /var/lib/apt/lists/* /tmp/* + +# Developers may want to add additional packages inside the image +# such as: telnet vimvim mc file + +# ================================================================== + +# Phase 4 - Install webwork2, pg, MathJaX which were downloaded to /opt/base/ in phase 1 +# Option: Install the OPL in the image also (about 850 MB) + +RUN mkdir -p $APP_ROOT/courses $APP_ROOT/libraries $APP_ROOT/libraries/webwork-open-problem-library $APP_ROOT/webwork2 /www/www/html + +COPY --from=base /opt/base/webwork2 $APP_ROOT/webwork2 +COPY --from=base /opt/base/pg $APP_ROOT/pg +COPY --from=base /opt/base/MathJax $APP_ROOT/MathJax + +# Optional - include OPL (also need to uncomment above to clone from GitHub when needed): +# ??? could/should this include the main OPL = /opt/base/webwork-open-problem-library/OpenProblemLibrary and not Contrib and Pending ??? +#COPY --from=base /opt/base/webwork-open-problem-library $APP_ROOT/libraries/webwork-open-problem-library + +# ================================================================== + +# Phase 5 - some configuration work + +# 1. Setup PATH. +# 2. Compiles color.c in the copy INSIDE the image, will also be done in docker-entrypoint.sh for externally mounted locations. +# 3. Some chown/chmod for material INSIDE the image. +# 4. Build some standard locales. +# 5. Set the default system timezone to be UTC. + +RUN echo "PATH=$PATH:$APP_ROOT/webwork2/bin" >> /root/.bashrc \ + && cd $APP_ROOT/pg/lib/chromatic && gcc color.c -o color \ + && cd $APP_ROOT/webwork2/ \ + && chown www-data DATA ../courses htdocs/applets logs tmp $APP_ROOT/pg/lib/chromatic \ + && chmod -R u+w DATA ../courses htdocs/applets logs tmp $APP_ROOT/pg/lib/chromatic \ + && echo "en_US ISO-8859-1\nen_US.UTF-8 UTF-8" > /etc/locale.gen \ + && /usr/sbin/locale-gen \ + && echo "locales locales/default_environment_locale select en_US.UTF-8\ndebconf debconf/frontend select Noninteractive" > /tmp/preseed.txt \ + && debconf-set-selections /tmp/preseed.txt \ + && rm /etc/localtime /etc/timezone && echo "Etc/UTC" > /etc/timezone \ + && dpkg-reconfigure -f noninteractive tzdata + +# These lines were moved into docker-entrypoint.sh so the bind mount of courses will be available #RUN cd $APP_ROOT/webwork2/courses.dist \ # && cp *.lst $APP_ROOT/courses/ \ # && cp -R modelCourse $APP_ROOT/courses/ -RUN cd $APP_ROOT/pg/lib/chromatic \ - && gcc color.c -o color +# ================================================================== + +# Phase 6 - install additional Perl modules from CPAN (not packaged for Ubuntu or outdated in Ubuntu) + +RUN cpanm install Statistics::R::IO \ + && rm -fr ./cpanm /root/.cpanm /tmp/* + +# Now installed from Ubuntu packages: +# XML::Parser::EasyTree Iterator Iterator::Util Pod::WSDL Array::Utils HTML::Template Mail::Sender Email::Sender::Simple Data::Dump +# For Ubuntu 16.04 would also need: +# Email::Address::XS + +# ================================================================== + +# Phase 7 - setup apache + +# Note we always create the /etc/ssl/local directory in case it will be needed, as +# the SSL config can also be done via a modified docker-entrypoint.sh script. + +# Always provide the dummy default-ssl.conf file: +COPY docker-config/ssl/default-ssl.conf /etc/apache2/sites-available/default-ssl.conf + +# However SSL will only be enabled at container startup via docker-entrypoint.sh. -# setup apache RUN cd $APP_ROOT/webwork2/conf \ && cp webwork.apache2.4-config.dist webwork.apache2.4-config \ && cp $APP_ROOT/webwork2/conf/webwork.apache2.4-config /etc/apache2/conf-enabled/webwork.conf \ @@ -133,30 +297,52 @@ RUN cd $APP_ROOT/webwork2/conf \ && a2enmod mpm_prefork \ && sed -i -e 's/Timeout 300/Timeout 1200/' /etc/apache2/apache2.conf \ && sed -i -e 's/MaxRequestWorkers 150/MaxRequestWorkers 20/' \ - -e 's/MaxConnectionsPerChild 0/MaxConnectionsPerChild 100/' \ - /etc/apache2/mods-available/mpm_prefork.conf \ + -e 's/MaxConnectionsPerChild 0/MaxConnectionsPerChild 100/' \ + /etc/apache2/mods-available/mpm_prefork.conf \ && cp $APP_ROOT/webwork2/htdocs/favicon.ico /var/www/html \ + && mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR $APACHE_LOG_DIR \ + && mkdir /etc/ssl/local \ + && a2enmod rewrite \ && sed -i -e 's/^$/\ - PerlPassEnv WEBWORK_URL\n\ - PerlPassEnv WEBWORK_ROOT_URL\n\ - PerlPassEnv WEBWORK_DB_DSN\n\ - PerlPassEnv WEBWORK_DB_USER\n\ - PerlPassEnv WEBWORK_DB_PASSWORD\n\ - PerlPassEnv WEBWORK_SMTP_SERVER\n\ - PerlPassEnv WEBWORK_SMTP_SENDER\n\ - PerlPassEnv WEBWORK_TIMEZONE\n\ - \n/' /etc/apache2/conf-enabled/webwork.conf - -RUN cd $APP_ROOT/webwork2/ \ - && chown www-data DATA ../courses htdocs/tmp htdocs/applets logs tmp $APP_ROOT/pg/lib/chromatic \ - && chmod -R u+w DATA ../courses htdocs/tmp htdocs/applets logs tmp $APP_ROOT/pg/lib/chromatic - -COPY docker-entrypoint.sh /usr/local/bin/ + PerlPassEnv WEBWORK_URL\n\ + PerlPassEnv WEBWORK_ROOT_URL\n\ + PerlPassEnv WEBWORK_DB_DSN\n\ + PerlPassEnv WEBWORK_DB_USER\n\ + PerlPassEnv WEBWORK_DB_PASSWORD\n\ + PerlPassEnv WEBWORK_SMTP_SERVER\n\ + PerlPassEnv WEBWORK_SMTP_SENDER\n\ + PerlPassEnv WEBWORK_TIMEZONE\n\ + \n/' /etc/apache2/conf-enabled/webwork.conf + +EXPOSE 80 +WORKDIR $APP_ROOT + +# Enabling SSL is NOT done here. +# Instead it is done by docker-entrypoint.sh at container startup when SSL=1 +# is set in the environment, for example by docker-compose.yml. +#RUN a2enmod ssl && a2ensite default-ssl +#EXPOSE 443 + +# ================================================================== + +# Phase 8 - prepare docker-entrypoint.sh +# Done near the end, so that an update to docker-entrypoint.sh can be +# done without rebuilding the earlier layers of the Docker image. + +COPY docker-config/docker-entrypoint.sh /usr/local/bin/ ENTRYPOINT ["docker-entrypoint.sh"] -EXPOSE 80 +# ================================================================== -WORKDIR $APP_ROOT +# Add enviroment variables to control some things during container startup + +ENV SSL=0 \ + PAPERSIZE=letter \ + SYSTEM_TIMEZONE=UTC \ + ADD_LOCALES=0 \ + ADD_APT_PACKAGES=0 + +# ================================================ CMD ["apache2", "-DFOREGROUND"] diff --git a/PG_VERSION b/PG_VERSION deleted file mode 100644 index c2a29f7f1a..0000000000 --- a/PG_VERSION +++ /dev/null @@ -1,4 +0,0 @@ -$PG_VERSION ='PG-2.14'; -$PG_COPYRIGHT_YEARS = '1996-2019'; - -1; diff --git a/README b/README index ca428fb85d..0777afbc52 100644 --- a/README +++ b/README @@ -3,8 +3,8 @@ Version 2.* Branch: github.com/openwebwork - http://webwork.maa.org/wiki/Release_notes_for_WeBWorK_2.13 - Copyright 2000-2017, The WeBWorK Project + http://webwork.maa.org/wiki/Release_notes_for_WeBWorK_2.14 + Copyright 2000-2019, The WeBWorK Project http://webwork.maa.org All rights reserved. diff --git a/VERSION b/VERSION index 7fa863cf31..e312985291 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,4 @@ - -$WW_VERSION = 'WeBWorK-2.14'; +$WW_VERSION = '2.15'; $WW_COPYRIGHT_YEARS = '1996-2019'; 1; diff --git a/bin/OPL-update b/bin/OPL-update index 72382f38cd..d9da002e66 100755 --- a/bin/OPL-update +++ b/bin/OPL-update @@ -21,7 +21,6 @@ use File::Basename; use Cwd; use DBI; - #(maximum varchar length is 255 for mysql version < 5.0.3. #You can increase path length to 4096 for mysql > 5.0.3) @@ -86,18 +85,34 @@ use WeBWorK::Utils::Tags; use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree/; my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}}); + +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = ($ce->{ENABLE_UTF8MB4})?1:0; +print "using utf8mb4 \n\n" if $ENABLE_UTF8MB4; + my $dbh = DBI->connect( $ce->{problemLibrary_db}->{dbsource}, $ce->{problemLibrary_db}->{user}, $ce->{problemLibrary_db}->{passwd}, { - PrintError => 0, - RaiseError => 1, + PrintError => 0, + RaiseError => 1, + ($ENABLE_UTF8MB4)?(mysql_enable_utf8mb4 =>1):(mysql_enable_utf8 => 1), }, ); +my $character_set=''; +$character_set=($ENABLE_UTF8MB4)?"utf8mb4":"utf8"; +$dbh->prepare("SET NAMES '$character_set'")->execute(); + +print "using character set $character_set to build OPL database\n"; + my $libraryRoot = $ce->{problemLibrary}->{root}; $libraryRoot =~ s|/+$||; +print "using libraryRoot $libraryRoot\n"; +print "WEBWORK_ROOT $ENV{WEBWORK_ROOT}\n"; my $libraryVersion = $ce->{problemLibrary}->{version}; my $db_storage_engine = $ce->{problemLibrary_db}->{storage_engine}; @@ -133,13 +148,13 @@ if($libraryVersion eq '2.5') { @create_tables = ( [$tables{dbsubject}, ' DBsubject_id int(15) NOT NULL auto_increment, - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, KEY DBsubject (name), PRIMARY KEY (DBsubject_id) '], [$tables{dbchapter}, ' DBchapter_id int(15) NOT NULL auto_increment, - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, DBsubject_id int(15) DEFAULT 0 NOT NULL, KEY DBchapter (name), KEY (DBsubject_id), @@ -147,7 +162,7 @@ if($libraryVersion eq '2.5') { '], [$tables{dbsection}, ' DBsection_id int(15) NOT NULL auto_increment, - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, DBchapter_id int(15) DEFAULT 0 NOT NULL, KEY DBsection (name), KEY (DBchapter_id), @@ -164,7 +179,7 @@ if($libraryVersion eq '2.5') { '], [$tables{path}, ' path_id int(15) NOT NULL auto_increment, - path varchar(255) NOT NULL, + path varchar(245) NOT NULL, machine varchar(255), user varchar(255), KEY (path), @@ -186,7 +201,7 @@ if($libraryVersion eq '2.5') { '], [$tables{keyword}, ' keyword_id int(15) NOT NULL auto_increment, - keyword varchar(256) NOT NULL, + keyword varchar(245) NOT NULL, KEY (keyword), PRIMARY KEY (keyword_id) '], @@ -210,7 +225,7 @@ if($libraryVersion eq '2.5') { chapter_id int (15) NOT NULL auto_increment, textbook_id int (15), number int(3), - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, page int(4), KEY (textbook_id, name), KEY (number), @@ -220,7 +235,7 @@ if($libraryVersion eq '2.5') { section_id int(15) NOT NULL auto_increment, chapter_id int (15), number int(3), - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, page int(4), KEY (chapter_id, name), KEY (number), @@ -237,7 +252,7 @@ if($libraryVersion eq '2.5') { '], [$tables{morelt}, ' morelt_id int(15) NOT NULL auto_increment, - name varchar(255) NOT NULL, + name varchar(245) NOT NULL, DBsection_id int(15), leader int(15), # pgfile_id of the MLT leader KEY (name), @@ -262,7 +277,7 @@ for my $tableinfo (@create_tables) { my $tabinit = $tableinfo->[1]; my $query = "DROP TABLE IF EXISTS `$tabname`"; $dbh->do($query); - $query = "CREATE TABLE `$tabname` ( $tabinit ) ENGINE=$db_storage_engine"; + $query = "CREATE TABLE `$tabname` ( $tabinit ) ENGINE=$db_storage_engine CHARACTER SET $character_set"; $dbh->do($query); if($lib eq 'OPL') { $old_tabname = $tabname; @@ -346,7 +361,7 @@ my ($name,$pgfile,$pgpath); #### First read in textbook information -if(open(IN, "$libraryRoot/Textbooks")) { +if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Textbooks")) { print "Reading in textbook data from Textbooks in the library $libraryRoot.\n"; my %textinfo = ( TitleText => '', EditionText =>'', AuthorText=>''); my $bookid = undef; @@ -445,10 +460,10 @@ my $tagtaxo = []; my ($chaplist, $seclist) = ([],[]); my $canopenfile = 0; -if(open(IN, "$libraryRoot/Taxonomy2")) { +if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy2")) { print "Reading in OPL taxonomy from Taxonomy2 in the library $libraryRoot.\n"; $canopenfile = 1; -} elsif(open(IN, "$libraryRoot/Taxonomy")) { +} elsif(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy")) { print "Reading in OPL taxonomy from Taxonomy in the library $libraryRoot.\n"; $canopenfile = 1; } else { @@ -516,6 +531,7 @@ if($canopenfile) { my $webwork_htdocs = $ce->{webwork_dir}."/htdocs"; my $file = "$webwork_htdocs/DATA/tagging-taxonomy.json"; open(OUTF, ">$file") or die "Cannot open $file"; +binmode(OUTF,':encoding(UTF-8)'); print OUTF to_json($tagtaxo,{pretty=>1}) or die "Cannot write to $file"; close(OUTF); print "Saved taxonomy to $file.\n"; @@ -970,6 +986,9 @@ if ($ce->{problemLibrary}{showLibraryLocalStats} || $ce->{problemLibrary}{showLibraryGlobalStats}) { print "\nUpdating Library Statistics.\n"; do $ENV{WEBWORK_ROOT}.'/bin/update-OPL-statistics'; + + print "\nLoading global statistics (if possible).\n"; + do $ENV{WEBWORK_ROOT}.'/bin/load-OPL-global-statistics.pl'; } diff --git a/bin/OPLUtils.pm b/bin/OPLUtils.pm old mode 100644 new mode 100755 index 323436ab48..0355322044 --- a/bin/OPLUtils.pm +++ b/bin/OPLUtils.pm @@ -89,7 +89,7 @@ sub build_library_directory_tree { # use the three arguments version of open # and check for errors - open $OUTFILE, '>', $file or die "Cannot open $file"; + open $OUTFILE, '>encoding(UTF-8)', $file or die "Cannot open $file"; # you can check for errors (e.g., if after opening the disk gets full) print { $OUTFILE } to_json(\@dirArray) or die "Cannot write to $file"; @@ -279,7 +279,7 @@ sub build_library_subject_tree { # use the three arguments version of open # and check for errors - open $OUTFILE, '>', $file or die "Cannot open $file"; + open $OUTFILE, '>encoding(UTF-8)', $file or die "Cannot open $file"; # you can check for errors (e.g., if after opening the disk gets full) print { $OUTFILE } to_json(\@subject_tree,{pretty=>1}) or die "Cannot write to $file"; diff --git a/bin/OPL_releases/Makefile b/bin/OPL_releases/Makefile new file mode 100644 index 0000000000..8e96915083 --- /dev/null +++ b/bin/OPL_releases/Makefile @@ -0,0 +1,21 @@ +RELEASE_FILENAME=webwork-open-problem-library-METADATA.tar.gz +GITHUB_USERNAME=heiderich +GITHUB_REPO=webwork-open-problem-library +JSON_FILENAME=latest_release.json +RELEASE_TAG=latest_release.tag + +${JSON_FILENAME}: + @echo "Download a JSON containing information on the latest release" + curl -s https://api.github.com/repos/${GITHUB_USERNAME}/${GITHUB_REPO}/releases/latest > ${JSON_FILENAME} + +${RELEASE_TAG}: ${JSON_FILENAME} + jq -r '.tag_name' ${JSON_FILENAME} > ${RELEASE_TAG} + +${RELEASE_FILENAME}: ${JSON_FILENAME} + curl -L `jq -r '[.assets | .[] | select(.name == "${RELEASE_FILENAME}" ) | .browser_download_url][0]' ${JSON_FILENAME}` -o ${RELEASE_FILENAME} + +extract: ${RELEASE_FILENAME} + tar xfz ${RELEASE_FILENAME} + +clean: + rm -f ${RELEASE_FILENAME} ${RELEASE_TAG} ${JSON_FILENAME} diff --git a/bin/OPL_releases/release.sh b/bin/OPL_releases/release.sh new file mode 100755 index 0000000000..24aca8b197 --- /dev/null +++ b/bin/OPL_releases/release.sh @@ -0,0 +1,369 @@ +#!/bin/bash +# +# Shell script to make releases of webwork problem libraries, heavily inspired +# by https://github.com/gap-system/ReleaseTools/blob/master/release + +set -e + +###################################################################### +# +# Usage information +# +help() { +cat < directory containing the library [Default: current directory] + --tmpdir to a temporary directory [Default: tmp subdirectory of current directory] + --libraryname library name + +Custom settings + -t, --tag git tag for the release + -r, --repository set GitHub repository (as USERNAME/REPONAME) + --token GitHub access token + +Notes: +* To learn how to create a GitHub access token, please consult + https://help.github.com/articles/creating-an-access-token-for-command-line-use/ +EOF + exit 0 +} + +###################################################################### +# +# Various little helper functions + + +# print notices in green +notice() { + printf '\033[32m%s\033[0m\n' "$*" +} + +# print warnings in yellow +warning() { + printf '\033[33mWARNING: %s\033[0m\n' "$*" +} + +# print error in red and exit +error() { + printf '\033[31mERROR: %s\033[0m\n' "$*" + exit 1 +} + +# check for uncommitted changes +verify_git_clean() { + git update-index --refresh + git diff-index --quiet HEAD -- || + error "uncommitted changes detected" +} + +# helper function for parsing GitHub's JSON output. Right now, +# we only extra the value of a single key at a time. This means +# we may end up parsing the same JSON data two times, but that +# doesn't really matter as it is tiny. +json_get_key() { + echo "$response" | python -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get("'"$1"'",""))' +} + +# On Mac OS X, tar stores extended attributes in ._FOO files inside archives. +# Setting COPYFILE_DISABLE prevents that. See +export COPYFILE_DISABLE=1 + + +###################################################################### +# +# Command line processing +# +CONTENT="METADATA" +LIBRARY_DIR="$PWD" +TMP_DIR="$PWD/tmp" + +FORCE=no +while [ x"$1" != x ]; do + option="$1" ; shift + case "$option" in + -h | --help ) help ;; + + --librarydir ) LIBRARY_DIR="$1"; shift ;; + --libraryname ) LIBRARY_NAME="$1"; shift ;; + --tmpdir ) TMP_DIR="$1"; shift ;; + + -t | --tag ) TAG="$1"; shift ;; + -r | --repository ) REPO="$1"; shift ;; + --token ) TOKEN="$1"; shift ;; + + -f | --force ) FORCE=yes ;; + --no-force ) FORCE=no ;; + + -- ) break ;; + * ) error "unknown option '$option'" ;; + esac +done + + +###################################################################### +# +# Some initial sanity checks +# + +command -v curl >/dev/null 2>&1 || + error "the 'curl' command was not found, please install it" + +command -v git >/dev/null 2>&1 || + error "the 'git' command was not found, please install it" + +command -v python >/dev/null 2>&1 || + error "the 'python' command was not found, please install it" + +cd $LIBRARY_DIR +#verify_git_clean + + +###################################################################### +# +# Determine the basename for the package archives +# +# + +notice "Library path: " $LIBRARY_DIR + +notice "Updating OPL tables" +OPL-update + +notice "Dumping OPL tables" +dump-OPL-tables.pl + +###################################################################### +# +# Fetch GitHub oauth token, used to authenticate the following commands. +# See https://help.github.com/articles/git-automation-with-oauth-tokens/ +# +if [ "x$TOKEN" = x ] ; then + TOKEN=$(git config --get github.token || echo) +fi +if [ "x$TOKEN" = x ] && [ -r ~/.github_shell_token ] ; then + TOKEN=$(cat ~/.github_shell_token) +fi +if [ "x$TOKEN" = x ] ; then + error "could not determine GitHub access token" +fi + + +###################################################################### +# +# Determine GitHub repository and username, and the current branch +# +if [ x"$REPO" = "x" ] ; then + error "could not guess GitHub repository" +fi +notice "Using GitHub repository $REPO" + +GITHUB_USER=$(dirname "$REPO") +notice "Using GitHub username $GITHUB_USER" + + +###################################################################### +# +# Derive API urls +# +API_URL=https://api.github.com/repos/$REPO/releases +UPLOAD_URL=https://uploads.github.com/repos/$REPO/releases + + +###################################################################### +# +# Determine the tag, and validate it +# +#verify_git_clean + +cd $LIBRARY_DIR + +if git show-ref -q "$TAG" ; then + notice "Using git tag $TAG" +else + notice "Creating git tag $TAG" + git tag "$TAG" +fi; + +HEAD_REF=$(git rev-parse --verify HEAD) +TAG_REF=$(git rev-parse --verify "$TAG^{}") + +if [ "x$TAG_REF" != "x$HEAD_REF" ] ; then + error "tag $TAG is not the HEAD commit -- did you tag the right commit?" +fi + + +echo "" + + +###################################################################### +# +# Get fresh (unmodified) copies of the files, and generate some stuff +# + +# Clean any remains of previous export attempts +mkdir -p "$TMP_DIR" +rm -rf "${TMP_DIR:?}/$LIBRARY_NAME"* + +# Set umask to ensure the file permissions in the release +# archives are sane. +umask 0022 + +notice "Preparing content" +mkdir "$TMP_DIR/$LIBRARY_NAME" +cp -r $LIBRARY_DIR/TABLE-DUMP "$TMP_DIR/$LIBRARY_NAME/TABLE-DUMP" +cp -r $LIBRARY_DIR/JSON-SAVED "$TMP_DIR/$LIBRARY_NAME/JSON-SAVED" + +#notice "Removing unnecessary files" +#rm -f .git* .hg* .cvs* +#rm -f .appveyor.yml .codecov.yml .travis.yml + + +###################################################################### +# +# Push commits to GitHub +# + +cd "$LIBRARY_DIR" + +# construct GitHub URL for pusing +REMOTE="https://$GITHUB_USER:$TOKEN@github.com/$REPO" + +# Make sure the branch is on the server +notice "Pushing your branch to GitHub" +notice "Remote: $REMOTE" + +# Make sure the tag is on the server +notice "Pushing your tag to GitHub" +if [ "x$FORCE" = xyes ] ; then + git push --force "$REMOTE" "$TAG" +else + git push "$REMOTE" "$TAG" +fi + +###################################################################### +# +# Create the GitHub release +# + +# check if release already exists +response=$(curl -s -S -X GET "$API_URL/tags/$TAG?access_token=$TOKEN") +MESSAGE=$(json_get_key message) +RELEASE_ID=$(json_get_key id) + +if [ "$MESSAGE" = "Not Found" ] ; then + MESSAGE= # release does not yet exist -> that's how we like it +elif [ x"$RELEASE_ID" != x ] ; then + # release already exists -> error out or delete it + if [ "x$FORCE" = xyes ] ; then + notice "Deleting existing release $TAG from GitHub" + response=$(curl --fail -s -S -X DELETE "$API_URL/$RELEASE_ID?access_token=$TOKEN") + MESSAGE= + else + error "release $TAG already exists on GitHub, aborting (use --force to override this)" + fi +fi + +if [ x"$MESSAGE" != x ] ; then + error "accessing GitHub failed: $MESSAGE" +fi + +# Create the release by sending suitable JSON +DATA=$(cat < "$ARCHIVENAME" + MIMETYPE="application/x-gzip" + ;; + .tar.bz2) + tar cf - "$LIBRARY_NAME" | bzip2 -9c > "$ARCHIVENAME" + MIMETYPE="application/x-bzip2" + ;; + .zip) + zip -r9 --quiet "$ARCHIVENAME" "$LIBRARY_NAME" + MIMETYPE="application/zip" + ;; + *) + warning "unsupported archive format $EXT" + continue + ;; + esac + if [ ! -f "$FULLNAME" ] ; then + error "failed creating $FULLNAME" + fi + notice "Uploading $ARCHIVENAME with mime type $MIMETYPE" + response=$(curl --fail --progress-bar -o "$TMP_DIR/upload.log" \ + -X POST "$UPLOAD_URL/$RELEASE_ID/assets?name=$ARCHIVENAME" \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $TOKEN" \ + -H "Content-Type: $MIMETYPE" \ + --data-binary @"$FULLNAME") +done + + + +exit 0 diff --git a/bin/addcourse b/bin/addcourse index c360808c47..111d256be2 100755 --- a/bin/addcourse +++ b/bin/addcourse @@ -1,8 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ -# $CVSHeader: webwork2/bin/addcourse,v 1.20 2006/12/09 03:29:56 sh002i Exp $ +# Copyright © 2000-2018 The WeBWorK Project, https://github.com/openwebwork # # This program is free software; you can redistribute it and/or modify it under # the terms of either: (a) the GNU General Public License as published by the diff --git a/bin/check_database_charsets.pl b/bin/check_database_charsets.pl new file mode 100755 index 0000000000..c6e0f8eeef --- /dev/null +++ b/bin/check_database_charsets.pl @@ -0,0 +1,10 @@ +#!/usr/bin/perl -w + +my $host = $ENV{WEBWORK_DB_HOST}; +my $port = $ENV{WEBWORK_DB_PORT}; +my $database_name = $ENV{WEBWORK_DB_NAME}; +my $database_user = $ENV{WEBWORK_DB_USER}; +my $database_password = $ENV{WEBWORK_DB_PASSWORD}; + + +print `mysql -u $database_user -p$database_password $database_name -h $host -e "SHOW VARIABLES WHERE Variable_name LIKE \'character\_set\_%\' OR Variable_name LIKE \'collation%\' or Variable_name LIKE \'init_connect\' "`; \ No newline at end of file diff --git a/bin/check_latex.tex b/bin/check_latex.tex index b75cb28180..0238df7a53 100644 --- a/bin/check_latex.tex +++ b/bin/check_latex.tex @@ -1,6 +1,6 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % WeBWorK Online Homework Delivery System -% Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +% Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ % $CVSHeader: webwork2/bin/check_latex.tex,v 1.2 2004/10/06 21:09:33 gage Exp $ % % This program is free software; you can redistribute it and/or modify it under diff --git a/bin/check_modules.pl b/bin/check_modules.pl index ab69cc9c78..80002c0f6f 100755 --- a/bin/check_modules.pl +++ b/bin/check_modules.pl @@ -37,13 +37,14 @@ Apache2::ServerUtil ); + + my @modulesList = qw( Array::Utils Benchmark Carp CGI Class::Accessor - #Crypt::SSLeay Dancer Dancer::Plugin::Database Data::Dump @@ -56,7 +57,7 @@ DBI Digest::MD5 Digest::SHA - Email::Address + Email::Address::XS Email::Simple Email::Sender::Simple Email::Sender::Transport::SMTP @@ -85,6 +86,8 @@ Locale::Maketext::Simple LWP::Protocol::https MIME::Base64 + Moo + MooX::Options Net::IP Net::LDAPS Net::OAuth @@ -113,6 +116,7 @@ UUID::Tiny XML::Parser XML::Parser::EasyTree + XML::Simple XML::Writer XMLRPC::Lite YAML diff --git a/bin/delcourse b/bin/delcourse index eb49d0c2c3..3c6e432066 100755 --- a/bin/delcourse +++ b/bin/delcourse @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/delcourse,v 1.4 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/dump-OPL-tables.pl b/bin/dump-OPL-tables.pl new file mode 100755 index 0000000000..d4ee88bc0f --- /dev/null +++ b/bin/dump-OPL-tables.pl @@ -0,0 +1,108 @@ +#!/usr/bin/perl + +############################################################################## +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2019 The WeBWorK Project, http://openwebwork.sf.net/ +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +############################################################################## + +# This script dumps the OPL library tables to a dump file. +use strict; + +# Get the necessary packages, including adding webwork to our path. + +BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n') + unless($ENV{WEBWORK_ROOT});} +use lib "$ENV{WEBWORK_ROOT}/lib"; + +use WeBWorK::CourseEnvironment; + +use String::ShellQuote; +use DBI; + +# get course environment and configured OPL path + +my $ce = new WeBWorK::CourseEnvironment({ + webwork_dir => $ENV{WEBWORK_ROOT}, + }); + +my $configured_OPL_path = $ce->{problemLibrary}{root}; + + +# Drop the "OpenProblemLibrary" from the end of the path + +$configured_OPL_path =~ s+OpenProblemLibrary++; + +# Check that it exists + +if ( -d "$configured_OPL_path" ) { + print "OPL path seems to be $configured_OPL_path\n"; +} else { + print "OPL path seems to be misconfigured as $configured_OPL_path which does not exist.\n"; + exit; +} + +# Set TABLE-DUMP path and make directory if necessary + +my $prepared_OPL_tables_dir = "${configured_OPL_path}/TABLE-DUMP"; +if ( ! -d "$prepared_OPL_tables_dir" ) { + `mkdir -p $prepared_OPL_tables_dir`; +} + +# Set dump file name + +my $prepared_OPL_tables_file = "$prepared_OPL_tables_dir/OPL-tables.sql"; + +# Get DB connection settings + +my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); + +$host = 'localhost' unless $host; + +$port = 3306 unless $port; + +my $dbuser = $ce->{database_username}; +my $dbpass = $ce->{database_password}; + +$dbuser = shell_quote($dbuser); +$dbpass = shell_quote($dbpass); +$db = shell_quote($db); + +$ENV{'MYSQL_PWD'}=$dbpass; + +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = $ce->{ENABLE_UTF8MB4}?1:0; + +my $character_set = ($ENABLE_UTF8MB4)? "utf8mb4":"utf8"; + +# Get mysqldump_command + +my $mysqldump_command = $ce->{externalPrograms}->{mysqldump}; + +# The tables to dump are: + +my $OPL_tables_to_dump = "OPL_DBsubject OPL_DBchapter OPL_DBsection OPL_author OPL_path OPL_pgfile OPL_keyword OPL_pgfile_keyword OPL_textbook OPL_chapter OPL_section OPL_problem OPL_morelt OPL_pgfile_problem"; + +# Tables NOT dumped: +# OPL_problem_user - is created by bin/update-OPL-statistics and need not be archived +# OPL_global_statistics - loaded from a special file provide by the OPL +# OPL_local_statistics - locally generated + +print "Dumping OPL tables\n"; + +`$mysqldump_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $db $OPL_tables_to_dump > $prepared_OPL_tables_file`; + +print "OPL database dump created: $prepared_OPL_tables_file\n"; + +1; diff --git a/bin/dump_past_answers b/bin/dump_past_answers index 6b45ae45a5..ace64481ee 100755 --- a/bin/dump_past_answers +++ b/bin/dump_past_answers @@ -2,7 +2,7 @@ ############################################################################## # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/fix_copyright.sh b/bin/fix_copyright.sh new file mode 100755 index 0000000000..a44a16034e --- /dev/null +++ b/bin/fix_copyright.sh @@ -0,0 +1,14 @@ +#!/bin/sh +sed -i .bak '/Copyright/c\ +# Copyright © 2000-2019. The WeBWorK Project. https://github.com/openwebwork/webwork2\ +' $1 + + + +#obtained by trial and error after much toil -- mostly error. +# this version works on a mac +# the space after -i might need to be removed for linux. +# produces $1.bak file + +# use with the find command: +# find . -name course.conf -exec /opt/webwork/webwork2/bin/fix_copyright.sh {} ';' \ No newline at end of file diff --git a/bin/integrity_check.pl b/bin/integrity_check.pl index 69bf341075..523e513029 100644 --- a/bin/integrity_check.pl +++ b/bin/integrity_check.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb_upgrade,v 1.17 2007/08/13 22:59:50 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/load-OPL-global-statistics.pl b/bin/load-OPL-global-statistics.pl new file mode 100755 index 0000000000..ad3bb62abc --- /dev/null +++ b/bin/load-OPL-global-statistics.pl @@ -0,0 +1,82 @@ +#!/usr/bin/perl + +############################################################################## +# WeBWorK Online Homework Delivery System +# Copyright © 2000-2019 The WeBWorK Project, http://openwebwork.sf.net/ +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of either: (a) the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version, or (b) the "Artistic License" which comes with this package. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See either the GNU General Public License or the +# Artistic License for more details. +############################################################################## + +# This script loads the OPL global statistics, which is often done by bin/update-OPL-statistics but may need to be done outside of that setting. +use strict; + +# Get the necessary packages, including adding webwork to our path. + +BEGIN{ die('You need to set the WEBWORK_ROOT environment variable.\n') + unless($ENV{WEBWORK_ROOT});} +use lib "$ENV{WEBWORK_ROOT}/lib"; + +use WeBWorK::CourseEnvironment; + +use String::ShellQuote; +use DBI; + +# get course environment and configured OPL path + +my $ce = new WeBWorK::CourseEnvironment({ + webwork_dir => $ENV{WEBWORK_ROOT}, + }); + +my $dbh = DBI->connect( + $ce->{problemLibrary_db}->{dbsource}, + $ce->{problemLibrary_db}->{user}, + $ce->{problemLibrary_db}->{passwd}, + { + AutoCommit => 0, + PrintError => 0, + RaiseError => 1, + }, +); + +# check to see if the global statistics file exists and if it does, upload it. + +my $global_sql_file = $ce->{problemLibrary}{root}.'/OPL_global_statistics.sql'; + +if (-e $global_sql_file) { + + my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); + + $host = 'localhost' unless $host; + + $port = 3306 unless $port; + + my $dbuser = $ce->{database_username}; + my $dbpass = $ce->{database_password}; + + + $dbh->do(<commit(); + + $dbuser = shell_quote($dbuser); + $dbpass = shell_quote($dbpass); + $db = shell_quote($db); + + $ENV{'MYSQL_PWD'}=$dbpass; + + my $mysql_command = $ce->{externalPrograms}->{mysql}; + + `$mysql_command --host=$host --port=$port --user=$dbuser $db < $global_sql_file`; + +} + +1; diff --git a/bin/newpassword b/bin/newpassword index 60c40818db..358a54262b 100755 --- a/bin/newpassword +++ b/bin/newpassword @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/newpassword,v 1.3 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/old_scripts/timing b/bin/old_scripts/timing index b44e004f55..64db42617f 100755 --- a/bin/old_scripts/timing +++ b/bin/old_scripts/timing @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/timing,v 1.5 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/old_scripts/wwaddindexing b/bin/old_scripts/wwaddindexing index a258d7edfa..7dfa7418a8 100755 --- a/bin/old_scripts/wwaddindexing +++ b/bin/old_scripts/wwaddindexing @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwaddindexing,v 1.3 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/old_scripts/wwdb_addgw b/bin/old_scripts/wwdb_addgw index 939bbf7c94..3ca4c12c1e 100755 --- a/bin/old_scripts/wwdb_addgw +++ b/bin/old_scripts/wwdb_addgw @@ -1,7 +1,7 @@ #!/usr/bin/perl -w ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb_addgw,v 1.3 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/old_scripts/wwdb_check b/bin/old_scripts/wwdb_check index a624523058..2b4e00207a 100755 --- a/bin/old_scripts/wwdb_check +++ b/bin/old_scripts/wwdb_check @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb_check,v 1.7 2006/08/14 17:23:56 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/old_scripts/wwdb_upgrade b/bin/old_scripts/wwdb_upgrade index 9a4b6ac859..208f33239b 100755 --- a/bin/old_scripts/wwdb_upgrade +++ b/bin/old_scripts/wwdb_upgrade @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb_upgrade,v 1.16 2007/07/22 05:24:20 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/pg-append-textbook-tags b/bin/pg-append-textbook-tags index 009ab087c9..952d8ff567 100755 --- a/bin/pg-append-textbook-tags +++ b/bin/pg-append-textbook-tags @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/pg-append-textbook-tags,v 1.1 2007/10/17 16:56:16 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under @@ -43,7 +43,7 @@ sub main { sub add_tags_to_file { my ($new_tags, $file) = @_; - my $pgfile = new IO::File($file, '+<:utf8') or do { + my $pgfile = new IO::File($file, '+ $ENV{WEBWORK_ROOT}, + }); + +my $configured_OPL_path = $ce->{problemLibrary}{root}; + + +# Drop the "OpenProblemLibrary" from the end of the path + +$configured_OPL_path =~ s+OpenProblemLibrary++; + +# Check that it exists + +if ( -d "$configured_OPL_path" ) { + print "OPL path seems to be $configured_OPL_path\n"; +} else { + print "OPL path seems to be misconfigured as $configured_OPL_path which does not exist.\n"; + exit; +} + +# Set TABLE-DUMP path and make directory if necessary + +my $prepared_OPL_tables_dir = "${configured_OPL_path}/TABLE-DUMP"; +if ( ! -d "$prepared_OPL_tables_dir" ) { + `mkdir -p $prepared_OPL_tables_dir`; +} + +# Set dump file name + +my $prepared_OPL_tables_file = "$prepared_OPL_tables_dir/OPL-tables.sql"; + +# Get DB connection settings + +my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); + +$host = 'localhost' unless $host; + +$port = 3306 unless $port; + +my $dbuser = $ce->{database_username}; +my $dbpass = $ce->{database_password}; + +$dbuser = shell_quote($dbuser); +$dbpass = shell_quote($dbpass); +$db = shell_quote($db); + +$ENV{'MYSQL_PWD'}=$dbpass; + +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = $ce->{ENABLE_UTF8MB4}?1:0; + +my $character_set = ($ENABLE_UTF8MB4)? "utf8mb4":"utf8"; + +my $mysql_command = $ce->{externalPrograms}->{mysql}; + +# check to see if the prepared_OPL_tables_file exists and if it does load it in + +if (-e $prepared_OPL_tables_file) { + `$mysql_command --host=$host --port=$port --user=$dbuser --default-character-set=$character_set $db < $prepared_OPL_tables_file`; +} + +1; diff --git a/bin/setfilepermissions b/bin/setfilepermissions index eb9ab9e03c..09ab0e5595 100755 --- a/bin/setfilepermissions +++ b/bin/setfilepermissions @@ -1,7 +1,7 @@ #!/usr/bin/env perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/newpassword,v 1.3 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/update-OPL-statistics b/bin/update-OPL-statistics.pl similarity index 81% rename from bin/update-OPL-statistics rename to bin/update-OPL-statistics.pl index 528b165f1c..b50f4855cc 100755 --- a/bin/update-OPL-statistics +++ b/bin/update-OPL-statistics.pl @@ -2,7 +2,7 @@ ############################################################################## # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under @@ -52,6 +52,12 @@ BEGIN webwork_dir => $ENV{WEBWORK_ROOT}, }); +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = $ce->{ENABLE_UTF8MB4}?1:0; + + my $dbh = DBI->connect( $ce->{problemLibrary_db}->{dbsource}, $ce->{problemLibrary_db}->{user}, @@ -66,7 +72,10 @@ BEGIN # get course list my @courses = listCourses($ce); -# create tables. We always redo the statistics table. +# create tables. We always redo the statistics table. + +my $character_set = ($ENABLE_UTF8MB4)? "utf8mb4":"utf8"; + $dbh->do(<do(<do(<commit(); @@ -180,35 +192,8 @@ BEGIN $dbh->commit(); -# check to see if the global statistics file exists and if it does, upload it. - -my $global_sql_file = $ce->{problemLibrary}{root}.'/OPL_global_statistics.sql'; - -if (-e $global_sql_file) { - - my ($dbi,$dbtype,$db,$host,$port) = split(':',$ce->{database_dsn}); - - $host = 'localhost' unless $host; - - $port = 3306 unless $port; - - my $dbuser = $ce->{database_username}; - my $dbpass = $ce->{database_password}; - - - $dbh->do(<commit(); - - $dbuser = shell_quote($dbuser); - $dbpass = shell_quote($dbpass); - $db = shell_quote($db); - - my $mysql_command = $ce->{externalPrograms}->{mysql}; - - `$mysql_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db < $global_sql_file`; - -} +# We no longer automatically load the global statistics data here. +print( "You may want to run load-OPL-global-statistics.pl to update the global statistics data.\n", + "If this is being run by OPL-update, that will be done automatically.\n"); 1; diff --git a/bin/upgrade_admin_db.pl b/bin/upgrade_admin_db.pl index 21b45ee6ed..73cc8166d7 100755 --- a/bin/upgrade_admin_db.pl +++ b/bin/upgrade_admin_db.pl @@ -2,7 +2,7 @@ ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/upload-OPL-statistics b/bin/upload-OPL-statistics.pl similarity index 95% rename from bin/upload-OPL-statistics rename to bin/upload-OPL-statistics.pl index 6f26e85ea9..37004296d3 100755 --- a/bin/upload-OPL-statistics +++ b/bin/upload-OPL-statistics.pl @@ -2,7 +2,7 @@ ############################################################################## # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under @@ -49,9 +49,11 @@ $dbpass = shell_quote($dbpass); $db = shell_quote($db); +$ENV{'MYSQL_PWD'}=$dbpass; + my $mysqldump_command = $ce->{externalPrograms}->{mysqldump}; -`$mysqldump_command --host=$host --port=$port --user=$dbuser --password=$dbpass $db OPL_local_statistics > $output_file`; +`$mysqldump_command --host=$host --port=$port --user=$dbuser $db OPL_local_statistics > $output_file`; print "Database File Created\n"; diff --git a/bin/wwapache2ctl.dist b/bin/wwapache2ctl.dist index 4b263b25f0..4a3e310b7e 100755 --- a/bin/wwapache2ctl.dist +++ b/bin/wwapache2ctl.dist @@ -1,7 +1,7 @@ #!/bin/sh ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwapache2ctl.dist,v 1.3 2007/08/13 22:59:50 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/wwapachectl.dist b/bin/wwapachectl.dist index 3a6b8a6cc6..83363e82d8 100755 --- a/bin/wwapachectl.dist +++ b/bin/wwapachectl.dist @@ -1,7 +1,7 @@ #!/bin/sh ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwapachectl.dist,v 1.17 2006/05/31 18:25:18 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/wwdb b/bin/wwdb index 834fadf86d..44aa435c00 100755 --- a/bin/wwdb +++ b/bin/wwdb @@ -1,7 +1,7 @@ #!/usr/bin/perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwdb,v 1.13 2006/01/25 23:13:45 sh002i Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/bin/wwsh b/bin/wwsh index 75c3ecfd7f..9219fec6b9 100755 --- a/bin/wwsh +++ b/bin/wwsh @@ -1,7 +1,7 @@ #!/usr/bin/perl ################################################################################ # WeBWorK Online Homework Delivery System -# Copyright © 2000-2007 The WeBWorK Project, http://openwebwork.sf.net/ +# Copyright © 2000-2018 The WeBWorK Project, http://openwebwork.sf.net/ # $CVSHeader: webwork2/bin/wwsh,v 1.10 2006/05/31 01:07:25 gage Exp $ # # This program is free software; you can redistribute it and/or modify it under diff --git a/clients/hello_world_apps/webwork_xmlrpc_client.pl b/clients/hello_world_apps/webwork_xmlrpc_client.pl index 8c30320c7c..1d838471ff 100755 --- a/clients/hello_world_apps/webwork_xmlrpc_client.pl +++ b/clients/hello_world_apps/webwork_xmlrpc_client.pl @@ -29,7 +29,7 @@ use constant REQUEST_URI => 'mod_xmlrpc'; use constant TEMPOUTPUTFILE => '/Users/gage/Desktop/renderProblemOutput.html'; -our $XML_URL = 'http://localhost:80'; +our $SITE_URL = 'http://localhost:80'; our $FORM_ACTION_URL = 'http://localhost:80/webwork2/html2xml'; our $XML_PASSWORD = 'xmlwebwork'; our $XML_COURSE = 'gage_course'; @@ -98,7 +98,7 @@ # Build client ############################################ our $xmlrpc_client = new WebworkClient ( - url => $XML_URL, + site_url => $SITE_URL, form_action_url => $FORM_ACTION_URL, displayMode => DISPLAYMODE(), diff --git a/clients/sendXMLRPC.pl b/clients/sendXMLRPC.pl index 68b6a8573a..16559e97ca 100755 --- a/clients/sendXMLRPC.pl +++ b/clients/sendXMLRPC.pl @@ -93,6 +93,7 @@ =head2 credentials file # site_url => 'http://localhost:80', # form_action_url => 'http://localhost:80/webwork2/html2xml', # site_password => 'xmlrpc', + # forcePortNumber => '80', # A port number to be forced, when needed. # Set the identification credential used by the "daemon_course" on the remote site courseID => "daemon_course", @@ -173,7 +174,7 @@ =head2 Options Same as -c but the question is rendered with the correct answers submitted. This succeeds only if the correct answers, as determined from the answer hash, all succeed. -=item f=s +=item -f formatName Specify the format used by the browser in displaying the question. Choices for s are @@ -182,6 +183,9 @@ =head2 Options debug simple +=item -l lang + + Set a language for the HTML rendering to use. Should use a value which would be valid for a course. =item -v @@ -201,6 +205,9 @@ =head2 Options Process question in TeX mode, convert to PDF and display. +=item --json + + Process question in JSON mode and save to file =item The single letter options can be "bundled" e.g. -vcCbB @@ -251,6 +258,10 @@ =head2 Options Sets problemSeed to the number contained in string s +=item --psvn=s + + Sets psvn to the number contained in string s + =back @@ -269,6 +280,7 @@ BEGIN } $ENV{MOD_PERL_API_VERSION} = 2; use lib "$main::dirname"; +print "home directory ".$main::dirname."\n"; #use lib "."; # is this needed? # some files such as FormatRenderedProblem.pm may need to be in the same directory @@ -290,11 +302,10 @@ BEGIN } } - +use lib "$WeBWorK::Constants::WEBWORK_DIRECTORY/lib"; use Carp; -#use Crypt::SSLeay; # needed for https use LWP::Protocol::https; use Time::HiRes qw/time/; use MIME::Base64 qw( encode_base64 decode_base64); @@ -314,7 +325,7 @@ BEGIN ### verbose output when UNIT_TESTS_ON =1; - our $UNIT_TESTS_ON = 0; +our $UNIT_TESTS_ON = 0; ############################################################ # Read command line options @@ -331,9 +342,11 @@ BEGIN my $verbose = ''; my $credentials_path; my $format = 'standard'; +my $lang = 'en'; my $edit_source_file = ''; my $display_tex_output=''; my $display_pdf_output=''; +my $display_json_output=''; my $print_answer_hash; my $print_answer_group; my $print_pg_hash; @@ -342,6 +355,7 @@ BEGIN my $read_list_from_this_file; my $path_to_log_file; my $problemSeed; +my $psvn; our %credentials; our @path_list; @@ -349,28 +363,31 @@ BEGIN GetOptions( - 'a' => \$display_ans_output1, - 'A' => \$display_ans_output2, - 'b' => \$display_html_output1, - 'B' => \$display_html_output2, - 'h' => \$display_hash_output1, - 'H' => \$display_hash_output2, - 'c' => \$record_ok1, # record_problem_ok1 needs to be written - 'C' => \$record_ok2, - 'v' => \$verbose, - 'e' => \$edit_source_file, - 'tex' => \$display_tex_output, - 'pdf' => \$display_pdf_output, - 'list=s' =>\$read_list_from_this_file, # read file containing list of full file paths - 'pg' => \$print_pg_hash, - 'anshash' => \$print_answer_hash, - 'ansgrp' => \$print_answer_group, + 'a' => \$display_ans_output1, + 'A' => \$display_ans_output2, + 'b' => \$display_html_output1, + 'B' => \$display_html_output2, + 'h' => \$display_hash_output1, + 'H' => \$display_hash_output2, + 'c' => \$record_ok1, # record_problem_ok1 needs to be written + 'C' => \$record_ok2, + 'v' => \$verbose, + 'e' => \$edit_source_file, + 'tex' => \$display_tex_output, + 'pdf' => \$display_pdf_output, + 'json' => \$display_json_output, + 'list=s' =>\$read_list_from_this_file, # read file containing list of full file paths + 'pg' => \$print_pg_hash, + 'anshash' => \$print_answer_hash, + 'ansgrp' => \$print_answer_group, 'resource' => \$print_resource_hash, - 'f=s' => \$format, + 'f=s' => \$format, + 'l=s' => \$lang, 'credentials=s' => \$credentials_path, 'help' => \$print_help_message, 'log=s' => \$path_to_log_file, 'seed=s' => \$problemSeed, + 'psvn=s' => \$psvn, ); print_help_message() if $print_help_message; @@ -396,9 +413,9 @@ BEGIN # credentials file location -- search for one of these files -our @path_list = ("$ENV{HOME}/.ww_credentials", "$ENV{HOME}/ww_session_credentials", 'ww_credentials', 'ww_credentials.dist'); +@path_list = ("$ENV{HOME}/.ww_credentials", "$ENV{HOME}/ww_session_credentials", 'ww_credentials', 'ww_credentials.dist'); -my $credentials_string = < 'http://localhost:80', # form_action_url => 'http://localhost:80/webwork2/html2xml', # site_password => 'xmlrpc', + # forcePortNumber => '80', # A port number to be forced, when needed. + # Set the identification credential used by the "daemon_course" on the remote site courseID => "daemon_course", @@ -562,9 +581,6 @@ BEGIN use constant DISPLAYMODE => 'MathJax'; use constant PROBLEMSEED => '987654321'; -### verbose output when UNIT_TESTS_ON =1; -our $UNIT_TESTS_ON = 0; - ############################################################ # End configure displays for local operating system ############################################################ @@ -592,7 +608,7 @@ BEGIN eval { # attempt to create log file local(*FH); - open(FH, '>>',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; + open(FH, '>>:encoding(UTF-8)',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; close(FH); }; @@ -608,17 +624,20 @@ BEGIN ############################################ my $default_input = { - userID => $credentials{userID}//'', - session_key => $credentials{session_key}//'', - courseID => $credentials{courseID}//'', - courseName => $credentials{courseID}//'', - course_password => $credentials{course_password}//'', - }; + userID => $credentials{userID}//'', + session_key => $credentials{session_key}//'', + courseID => $credentials{courseID}//'', + courseName => $credentials{courseID}//'', + course_password => $credentials{course_password}//'', +}; my $default_form_data = { - displayMode => $DISPLAYMODE, - outputformat => $format//'standard', - problemSeed => $problemSeed//PROBLEMSEED(), + displayMode => $DISPLAYMODE, + outputformat => $format//'standard', + problemSeed => $problemSeed//PROBLEMSEED(), + psvn => $psvn//'23456', + forcePortNumber => $credentials{forcePortNumber}//'', + language => $lang//'en', }; ################################################## @@ -718,6 +737,19 @@ sub process_pg_file { system($PDF_DISPLAY_COMMAND." ".$pdf_path); } } + if ($display_json_output) { + my $form_data2 = { + %$form_data1, + outputformat => 'json', + displayMode =>'MathJax', + }; + print "Creating json\n" if $UNIT_TESTS_ON; + my ($error_flag, $formatter, $error_string) = + process_problem($file_path, $default_input, $form_data2); + my $json_file_name = create_json_output($file_path, $formatter); + print( "Created JSON data in file ", TEMPOUTPUTDIR(), $json_file_name, "\n"); + exit; + } my ($error_flag, $formatter, $error_string) = process_problem($file_path, $default_input, $form_data1); # extract and display result @@ -845,7 +877,7 @@ sub process_problem { ### build client my $xmlrpc_client = new WebworkClient ( - url => $credentials{site_url}, + site_url => $credentials{site_url}, form_action_url => $credentials{form_action_url}, site_password => $credentials{site_password}//'', courseID => $credentials{courseID}, @@ -862,10 +894,13 @@ sub process_problem { my $problemSeed = $form_data->{problemSeed}; die "problem seed not defined in sendXMLRPC::process_problem" unless $problemSeed; + + my $local_psvn = $form_data->{psvn}//34567; my $updated_input = {%$input, envir => $xmlrpc_client->environment( fileName => $adj_file_path, sourceFilePath => $adj_file_path, + psvn => $local_psvn, problemSeed => $problemSeed,), }; @@ -928,7 +963,7 @@ sub process_problem { WebworkClient::writeRenderLogEntry("", "{script:$scriptName; file:$file_path; ". sprintf("duration: %.3f sec;", $cg_duration). - " url: $credentials{site_url}; }",''); + " site_url: $credentials{site_url}; }",''); ####################################################################### # End processing of the pg file @@ -1103,7 +1138,7 @@ sub create_tex_output { $file_name =~ s/\.\w+$/\.tex/; # replace extension with tex my $output_file = TEMPOUTPUTDIR().$file_name; local(*FH); - open(FH, '>', $output_file) or die "Can't open file $output_file for writing"; + open(FH, '>:encoding(UTF-8)', $output_file) or die "Can't open file $output_file for writing"; print FH $output_text; close(FH); print "tex result sent to $output_file\n" if $UNIT_TESTS_ON; @@ -1112,6 +1147,25 @@ sub create_tex_output { return $file_name; } +sub create_json_output { + my $file_path = shift; + my $formatter = shift; + my $output_text = $formatter->formatRenderedProblem; + $file_path =~s|/$||; # remove final / + $file_path =~ m|/?([^/]+)$|; + my $file_name = $1; + $file_name =~ s/\.\w+$/\.json/; # replace extension with json + my $output_file = TEMPOUTPUTDIR().$file_name; + local(*FH); + open(FH, '>:encoding(UTF-8)', $output_file) or die "Can't open file $output_file for writing"; + print FH $output_text; + close(FH); + print "json result sent to $output_file\n" if $UNIT_TESTS_ON; +# sleep 5; #wait 5 seconds +# unlink($output_file); + return $file_name; +} + sub display_html_output { #display the problem in a browser my $file_path = shift; my $formatter = shift; @@ -1122,7 +1176,7 @@ sub create_tex_output { $file_name =~ s/\.\w+$/\.html/; # replace extension with html my $output_file = TEMPOUTPUTDIR().$file_name; local(*FH); - open(FH, '>', $output_file) or die "Can't open file $output_file for writing"; + open(FH, '>:encoding(UTF-8)', $output_file) or die "Can't open file $output_file for writing"; print FH $output_text; close(FH); @@ -1211,7 +1265,7 @@ sub record_problem_ok1 { } local(*FH); - open(FH, '>>',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; + open(FH, '>>:encoding(UTF-8)',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; print FH $return_string; close(FH); return $SHORT_RETURN_STRING; @@ -1234,7 +1288,7 @@ sub record_problem_ok2 { $all_correct = ".5" if $some_correct_answers_not_specified; $ALL_CORRECT = ($all_correct == 1)?'All answers are correct':'Some answers are incorrect'; local(*FH); - open(FH, '>>',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; + open(FH, '>>:encoding(UTF-8)',$path_to_log_file) or die "Can't open file $path_to_log_file for writing"; print FH "$all_correct $file_path\n"; # do we need this? compile_errors=$error_flag\n"; close(FH); return $ALL_CORRECT; @@ -1275,7 +1329,15 @@ sub get_source { if ($file_path eq '-') { $source = ; } else { - open(FH, '<',$file_path) or die "Couldn't open file $file_path: $!"; + # To support proper behavior with UTF-8 files, we need to open them with "<:encoding(UTF-8)" + # as otherwise, the first HTML file will render properly, but when "Preview" "Submit answer" + # or "Show correct answer" is used it will make problems, as in process_problem() the + # encodeSource() method is called on a data which is still UTF-8 encoded, and leads to double + # encoding and gibberish. + # NEW: + open(FH, "<:encoding(UTF-8)" ,$file_path) or die "Couldn't open file $file_path: $!"; + # OLD: + #open(FH, "<" ,$file_path) or die "Couldn't open file $file_path: $!"; $source = ; #slurp input close FH; } @@ -1377,12 +1439,12 @@ sub print_help_message { or create a file with this information and specify it with the --credentials option. %credentials = ( - userID => "my login name for the webwork course", - course_password => "my password ", - courseID => "the name of the webwork course", - XML_URL => "url of rendering site - XML_PASSWORD => "site password" # preliminary access to site - $FORM_ACTION_URL = 'http://localhost:80/webwork2/html2xml'; #action url for form + userID => "my login name for the webwork course", + course_password => "my password ", + courseID => "the name of the webwork course", + SITE_URL => "url of rendering site", + XML_PASSWORD => "site password", # preliminary access to site + form_action_url => 'http://localhost:80/webwork2/html2xml' #action url for form ); Options diff --git a/clients/t/Second_semester_calculus_topics.html b/clients/t/Second_semester_calculus_topics.html deleted file mode 100644 index d996f55181..0000000000 --- a/clients/t/Second_semester_calculus_topics.html +++ /dev/null @@ -1,327 +0,0 @@ - - - - - - mth162_overview - - - - - - - - -
-

Math 162, Second semester Calculus

- -

Here are some remarks on the topics we will cover this semester with examples
-of the kinds of homework questions we’ll ask you to work on in each homework set.

- -

Review: 0. Fundamental theorem of Calculus, Integration by Substitution, Indefinite integration

-

-Be patient while the entire page of problems loads for the first time. -

-

-Additional information about WeBWorK can be found at https://webwork.maa.org/wiki and https://webwork.maa.org/planet -

- -

- -

Play with the Geogebra applet above to get a feeling for the relationship between \(f\), the height
-of the upper boundary of the region at \(x\), and \(F\), the area of the region to the left of \(x\).

- -

\(f\) is the derivative of \(F\) and \(F\) is an anti-derivative of \(f\). A visual image of this
-relationship is useful. (Why did I write an antiderivative?)

- -

- -

- -

1. Topics: , Areas Between Curves

- -

- -

The example above is simple once you draw the picture! We’ll see how
-this can be reduced to integrating \(f(x)-g(x)\). To make this visually
-obvious imagine that the region is composed of vertical tooth picks and
-then slide them so their butts all rest on the \(x\)-axis. The tooth picks
-haven’t changed length or thickness so the area hasn’t changed either.
-(look up Cavalieri and his principle !)

- -

2. Topics: Volumes, Work

- -

- -

Visualizing and describing the region whose area or volume you want to find is often the hardest part of the problem. You’ll get lots of practice visualizing as with the example above and the one below.

- -

- -

This problem is harder (at first). You first have to visualize the region, then understand the volume of each of the thin vertical cylinders it’s made up of, (cylinders are what you get if you rotate vertical toothpicks around the \(y\) axis), and then add these
-volumes together (the Riemann sum).

- -

The Riemann integral is simply the ‘one and only’ value (mathematicians say ‘unique’ value) you get when the toothpicks are thin enough. BUT the integral has the enormous advantage that you can calculate it using anti-derivatives instead of doing lots and lots of addition of volumes.

- -

3. Topics: Integration by Parts, Trig Integrals

- -

In some ways this section is more technical – a bunch of challenges, like solving cross-word puzzles – some people find this really fun.
-As an analyst and a geometer I can tell you that integration by parts also has REALLY important theoretical uses as well.
-I can’t explain the theoretical importance to you yet however, maybe next semester if we get to Stoke’s theorem in multivariable calculus. :-)

- -

4. Topics: Trig Substitution, Partial Fractions

- -

More technical challenges. For me, when I took calculus, these challenges were one of the most fun parts of the course.

- -

But I have admit, particularly now-a-days, that Mathematica or Maple or Sage (or you guys) can solve these faster than I can.

- -

These challenges do illustrate one larger concept however. These antidifferentiation techniques are the simplest example of Inverse Problems. We can differentiate most functions \(F\) mechanically –it’s much harder to do the reverse – to guess the function \(F\) given the function \(f\) (and then check your guess by differentiating \(F\)). And yes guessing (then checking) plays a big role in Inverse Problems.

- -

“Integration techniques” are just advanced, efficient methods of guessing of anti-derivatives. (And usually it is worth your while to validate your answer by differentiating.

- -

Another inverse problem is solving differential equations, and inverse problems just get more and more interesting from there.

- -

Given the shape of a drum one can calculate what it will sound like relatively easily. If you hear a drum can you predict its shape from the sound alone? Well?? What do you think?

- -

5 Topics: Improper Integrals, Arc Length

- -

These two topics don’t fit together that well. We’ll talk about about improper integrals more later. The arc length problem is simply calculating the length of a rope when it is not straight. You do it by approximating by straight lines and then adding everything up of course. Here is an example.

- -

6. Topics: Surface Area, Curves defined by parametric equations

- -

7. Topics: Calculus of parametric curves

- -

This is the beginning of a topic which is really important to physicists and engineers – how to describe things that don’t go in straight lines. It’s also important for video game designers (oh yes and mathematicians like them as well.) Here is a sample problem – don’t try to calculate the area yet just see why the formula makes sense. (You can try visualizing as the sum of two vectors that change with time.)

- -

- -

8. Topics: Polar coordinates: Areas and lengths, Sequences

- -

Physicists and differential geometers have a magic weapon. They choose the right coordinate system to describe the problem. If you get the coordinate system right the problem becomes much simpler. Here is an example.
-

- -

9 Topics: Series, Integral test

- -

Now we get to series. A topic that most students think is impossible at first and then up wondering what they thought was so hard about it. The hidden trick here – which is not immediately apparent from the cookbook methods – is the art of approximation – finding inequalities that give you useful information.
-This is not widely taught in high schools but its a very creative and interesting skill to acquire – usually through lots of practice. The squeeze principle which you used in the first semester to find limits is an example of this type of thinking. Here is an example of a series question.

- -

Below is an example (one from the power series chapter) where you can glimpse it’s importance. A power series defines a function which is just a really long polynomial. If the first 10 decimal points of \(f\)
-are completely determined by say the first 20 terms of the series then in that case you might as well just deal with the polynomial, which you understand concretely (and computers can calculate with). “Functions are just long polynomials.” This isn’t strictly true but it’s 90% true and thinking this way gets you started in the right direction.

- -

To get started solving the problem below just plug in some values. You can use the convention that \(0^0=1\) –it’s defined that way because it makes these sigma summation formulas easy to write.

- -

Don’t be worried by the sigma notation – get someone to show you how to write it out in long string … or look it up on the web.
-

- -

10 Topics: Comparison tests, Alternating series

- -

Positive series (where each term is positive) have two choices. Each term is like a derivative (the speed) and the sum is how far you have traveled. Since the speed is positive either you travel all the way to infinity along the \(x\) axis or you travel up to some finite bound (Think \(1/2 + 1/4 +1/8 +...\) can you ever get beyond \(1\) when each term is slowing down that fast? Once you realize this, the trick is to find a known series that goes only a finite distance but is larger than your series (which proves your series converges) or find a known series that is smaller and travels off to infinity, pushing your series ahead of it. Everything else is just details of finding one or the other of these known series.

- -

11 Topics: Ratio and Root tests, Absolute convergence

- -

These are the details – we’ll talk about them when the time comes.

- -

12 Topics: Power series

- -

More details – specialized for power series “really long polynomials”.

- -

13 Topics: Representation of functions as power series, Taylor series

- -

As I said above there is a sense in which every reasonable function is just a long polynomials. Talyor series come at the end of this course but they are the foundation of most applications and a lot of theory. Just ask anyone taking MTH280 (numerical analysis) or MTH285(applied mathematics) or MTH282 (Complex analysis). They model reality, they solve differential equations. Of course there are a few functions which are much weirder and don’t behave much like long polynomials and mathematicians (and some physicists) get excited about understanding those if for no other reason that they are a challenge to figure out.

- -

Topics: Application of power series

- -

Power series are very useful. Here is an example.
-

- -

For the JMM presentation

- -

Here is the magic code for embedding these problems.

-
&lt; iframe  width="800" height="400" 
-src="https://demo.webwork.rochester.edu/webwork2/html2xml?
-	&answersSubmitted=0&
-	&sourceFilePath=Library/Union/setSeriesTaylor/ur_sr_9_6.pg&
-	&problemSeed=123567&
-	&courseID=daemon_course&
-	&userID=daemon&
-	&course_password=daemon&
-	&showSummary=1&
-	&displayMode=MathJax&
-	&problemIdentifierPrefix=102&
-	&language=en&
-	&outputformat=sticky">
-&lt; /iframe>
-

-
-
-
-
- - -
- - \ No newline at end of file diff --git a/clients/t/show-psvn.pg b/clients/t/show-psvn.pg new file mode 100644 index 0000000000..495f54bc81 --- /dev/null +++ b/clients/t/show-psvn.pg @@ -0,0 +1,19 @@ +##DESCRIPTION +## A very simple first problem +##ENDDESCRIPTION +##KEYWORDS('algebra') +DOCUMENT(); # This should be the first executable line in the problem. +loadMacros( +"PG.pl", +"PGbasicmacros.pl", +); + +TEXT(beginproblem()); + +BEGIN_TEXT +$PAR +PSVN is $psvn +$PAR +END_TEXT + +ENDDOCUMENT(); # This should be the last executable line in the problem. diff --git a/clients/t/test-utf8-hebrew.pg b/clients/t/test-utf8-hebrew.pg new file mode 100644 index 0000000000..b932d046f0 --- /dev/null +++ b/clients/t/test-utf8-hebrew.pg @@ -0,0 +1,37 @@ +DOCUMENT(); # This should be the first executable line in the problem. + +loadMacros( + "PGstandard.pl", + "MathObjects.pl", +); + +SET_PROBLEM_TEXTDIRECTION("rtl"); +SET_PROBLEM_LANGUAGE("he-IL"); + +TEXT(beginproblem()); + +Context("Numeric"); + +$a = random(1,9,1); +$b = random(1,9,1); + +$ans1 = Real("10*$a + $b"); + +Context()->texStrings; +BEGIN_TEXT + +חשב ×ת \( f(x) = 10 x + $b \) עבור הערך \( x = $a \). +$PAR + +$BCENTER +\{ openSpan( { "dir" => "ltr" } ) \} +\( f($a) = \) \{ $ans1->ans_rule(2) \} +\{ closeSpan() \} +$ECENTER + +END_TEXT +Context()->normalStrings; + +ANS( $ans1->cmp() ); + +ENDDOCUMENT(); # This should be the last executable line in the problem. diff --git a/clients/t/Second_semester_calculus_topics2.html b/clients/t/test_embedded_problems.html similarity index 98% rename from clients/t/Second_semester_calculus_topics2.html rename to clients/t/test_embedded_problems.html index d996f55181..9b28eedc11 100644 --- a/clients/t/Second_semester_calculus_topics2.html +++ b/clients/t/test_embedded_problems.html @@ -67,7 +67,7 @@


@@ -88,7 +88,7 @@


@@ -103,7 +103,7 @@


@@ -120,7 +120,7 @@

1. Topics: , Areas Between &course_password=daemon& &showSummary=1& &displayMode=MathJax& - &problemIdentifierPrefix=102& + &problemUUID=102& &language=en& &outputformat=sticky">

@@ -144,7 +144,7 @@

2. Topics: Volumes, Work

&course_password=daemon& &showSummary=1& &displayMode=MathJax& - &problemIdentifierPrefix=102& + &problemUUID=102& &language=en& &outputformat=sticky">

@@ -161,7 +161,7 @@

2. Topics: Volumes, Work

&course_password=daemon& &showSummary=1& &displayMode=MathJax& - &problemIdentifierPrefix=102& + &problemUUID=102& &language=en& &outputformat=sticky">

@@ -211,7 +211,7 @@

7. Topics: Calculu &course_password=daemon& &showSummary=1& &displayMode=MathJax& - &problemIdentifierPrefix=102& + &problemUUID=102& &language=en& &outputformat=sticky">

@@ -229,7 +229,7 @@

8. Topics

@@ -255,7 +255,7 @@

9 Topics: Series, Integral &course_password=daemon& &showSummary=1& &displayMode=MathJax& - &problemIdentifierPrefix=102& + &problemUUID=102& &language=en& &outputformat=sticky">

@@ -289,7 +289,7 @@

Topics: Application of &course_password=daemon& &showSummary=1& &displayMode=MathJax& - &problemIdentifierPrefix=102& + &problemUUID=102& &language=en& &outputformat=sticky">

@@ -307,7 +307,7 @@

For the JMM presentation

&course_password=daemon& &showSummary=1& &displayMode=MathJax& - &problemIdentifierPrefix=102& + &problemUUID=102& &language=en& &outputformat=sticky"> &lt; /iframe> diff --git a/clients/t/test_formats_WWproblems.html b/clients/t/test_formats_WWproblems.html new file mode 100644 index 0000000000..1f37c23dbf --- /dev/null +++ b/clients/t/test_formats_WWproblems.html @@ -0,0 +1,106 @@ + + + + + test formats for embedded problems + + + + +

Format: sticky

+

+

Format: standard (has error section -- but there should be no errors)

+

+ +

Format: simple

+

+ +

Format: body_text

+

+ +

Format: PTX

+

+ +

Format: JSON

+

+ + + diff --git a/t/simpleTemplateTest.html.dist b/clients/t/test_simpleFormat_WWproblem.html similarity index 91% rename from t/simpleTemplateTest.html.dist rename to clients/t/test_simpleFormat_WWproblem.html index dff7ce8330..079cd45992 100644 --- a/t/simpleTemplateTest.html.dist +++ b/clients/t/test_simpleFormat_WWproblem.html @@ -35,7 +35,7 @@

Problem 1-- xmlrpc version of a simple template

diff --git a/t/EmbeddedWWProblems.html b/clients/t/test_toggled_embedded_WWproblems.html similarity index 89% rename from t/EmbeddedWWProblems.html rename to clients/t/test_toggled_embedded_WWproblems.html index c02183ecc0..2587e2726f 100644 --- a/t/EmbeddedWWProblems.html +++ b/clients/t/test_toggled_embedded_WWproblems.html @@ -28,6 +28,19 @@ + + +

Embedded WeBWorK Problems

Problem 1 -- interval notation @@ -38,7 +51,7 @@

Embedded WeBWorK Problems

- - +
+
+ Click here to toggle problem display: (PGinfo example) +
+