Flatpak stuff

- make R part more user friendly - update renv version - more user friendly and get rid of forced lowercase - got debug symbols - update README a bit - improve caching slightly + make working directory check more lenient - Also updated appdata from tryFixFlatpak branch
jasp-stats · Sep 21, 2021 · a22bb90 · a22bb90
1 parent dab57f0
commit a22bb90
Show file tree

Hide file tree

Showing 11 changed files with 439 additions and 4,250 deletions.
diff --git a/Tools/flatpak/RPackages.json b/Tools/flatpak/RPackages.json
diff --git a/Tools/flatpak/org.jaspstats.JASP.appdata.xml b/Tools/flatpak/org.jaspstats.JASP.appdata.xml
@@ -35,6 +35,68 @@
 	<url type="bugtracker">https://github.com/jasp-stats/jasp-issues/issues</url>
 	<url type="donation">https://jasp-stats.org/donate/</url>
 	<releases>
+		<release version="0.15.0" date="2021-09-21">
+					<description>
+						New release 0.15
+
+						New features and improvements
+
+						- Plot editing
+						-- Allow custom fonts in plots
+						-- Resized plots are kept after refreshing 
+						- Supports more languages, English, German, Dutch, Galician, Japanese, Portuguese and rudimentary Chinese and Spanish
+						- Windows workarounds:
+						-- Support reading "native" locale encoded csvs and utf-8 both, dependent on BOM. (Aka save as csv vs csv with utf-8 in excel)
+						-- Support installing jasp for user with username in native locale characters
+						- New module “Prophet” for time series prediction
+						- Descriptives: 
+						-- Dot plots
+						-- Stem-and-leaf tables
+						-- Descriptives table can now be transposed
+						- SEM: 
+						-- Possibility to define several models
+						-- MIMIC model analysis
+						-- Latent Growth analysis
+						-- Improved labels in tables
+						-- Footnote fit-table listwise deletion
+						- T-Tests and ANOVAs: Raincloud plots
+						- Distributions: Major update including 23 new continuous and 5 discrete distributions
+						- Learn Bayes:
+						-- Binary classification
+						-- Game of chance
+						-- Game of skill
+						-- Buffon’s needle
+						-- Some edge case scenario and esthetical fixes for binomial estimation
+						- Regression:
+						-- Bootstrapping in correlation
+						-- Categorical variables can be used as predictors in linear regression
+						-- Adjusted lower limit for wls weights 
+						-- Relaxed error check on the covariance matrix of predictors 
+						- Frequencies:
+						- Show more informative footnote when phi cannot be calculated
+						-- Choose odds ratio for contingency table
+						-- Guttman and Kruskal’s lambda for contingency tables
+						- Factor analysis: Factor loadings can now be sorted in EFA and CFA
+						- Mixed Models
+						-- Fixing plotting themes
+						-- Improving warning messages
+						-- Changing default links for gamma and negative binomial family
+						- Meta-Analyses - Complete update of the RoBMA module
+						- Reliability
+						-- Classical intraclass correlation coefficients 
+						-- Adding options for mean and SD
+						-- Restructuring of R-code to improve computational time and reduce error-proneness
+						-- Manage huge .jasp file sizes produced by sampling in reliability analysis
+						- Audit - Added mean and median to descriptive output of prior and posterior distribution
+
+					Bug fixes
+						- Several fixes to sav loading (spss files)
+						- Loading data repeated measures ANOVA
+						- Cohen’s d was incorrectly calculated
+						- Fixed error in density graph Bayesian meta-analysis
+						- Fixed error in computing the total number of edges in Network analysis
+			</description>
+		</release>
 		<release version="0.14.1" date="2020-10-14">
 					<description>
 						New release 0.14

diff --git a/Tools/flatpak/org.jaspstats.JASP.json b/Tools/flatpak/org.jaspstats.JASP.json
@@ -1,8 +1,10 @@
 {
 	"app-id":			"org.jaspstats.JASP",
 	"runtime": 			"org.kde.Platform",
-	"runtime-version": 		"5.12",
+	"runtime-version":	"5.15",
 	"sdk": 				"org.kde.Sdk",
+	"base":				"io.qt.qtwebengine.BaseApp",
+	"base-version":		"5.15",
 	"command": 			"org.jaspstats.JASP",
 	"finish-args": [
 		"--socket=x11",
@@ -13,38 +15,45 @@
 		"--device=dri"
 	],
 	"build-options": {
-		"env": {
-			"CXX11":		"g++",
-			"CXX14":		"g++",
-			"CXX11FLAGS":	"-std=gnu++11",
-			"CXX14FLAGS":	"-std=gnu++14"
-		},
 		"cxxflags": 				"-O2 -g -Wno-error",
 		"cflags": 					"-Wno-error",
 		"ldflags":					"-lgfortran",
 		"append-ld-library-path": 	"/app/lib64/R/lib",
 		"env": {
-			"GIT_DISCOVERY_ACROSS_FILESYSTEM": "true",
-			"R_HOME": "/app/lib64/R/",
-			"PREFIX": "/app"
+
+            "CXX11":		"g++",
+			"CXX14":		"g++",
+			"CXX11FLAGS":	"-std=gnu++11",
+			"CXX14FLAGS":	"-std=gnu++14",
+
+			"JASP_R_REPOSITORY":				"file:///app/lib64/local-cran",
+			"GITHUB_PAT_DEF":					"Z2hwX0M3YnZhaEJIYjF5QVVxN0p5NmtjTXhpcWRveXIyOTEwOXc4Rwo",
+			"GITHUB_PAT":						"Z2hwX1FVQmt2dWk0WFV5SWJrN0VKc2JUWWVnTzFaVnQxbzROWmxwdwo",
+
+			"GIT_DISCOVERY_ACROSS_FILESYSTEM":	"true",
+
+			"PREFIX": 							"/app",
+			"R_HOME": 							"/app/lib64/R/",
+
+			"QMAKEPATH":						"/app/lib"
 		}
 	},
 	"cleanup": [
 		"/include"
 	],
-	"modules":[	
-		{ 
-			"name": "boost",
-			"buildsystem": "simple",
+	"modules":[
+		{
+			"name": 		"boost",
+			"buildsystem": 	"simple",
 			"build-commands":[
 				"./bootstrap.sh --with-libraries=filesystem,system",
 				"./b2 -j${FLATPAK_BUILDER_N_JOBS} install --prefix=/app" 
 			],
 			"sources":[
 				{
-					"type": "archive",
-					"url": "https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz",
-					"sha256": "96b34f7468f26a141f6020efb813f1a2f3dfb9797ecf76a7d7cbd843cc95f5bd"
+					"type": 	"archive",
+					"url": 		"https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz",
+					"sha256": 	"96b34f7468f26a141f6020efb813f1a2f3dfb9797ecf76a7d7cbd843cc95f5bd"
 				}
 			]
 		},
@@ -62,11 +71,11 @@
 			"sources": [
 				{
 					"type":   "archive",
-					"url":    "http://cran.r-project.org/src/base/R-3/R-3.6.1.tar.gz",
-					"sha256": "5baa9ebd3e71acecdcc3da31d9042fb174d55a42829f8315f2457080978b1389"
+					"url":    "http://cran.r-project.org/src/base/R-4/R-4.1.0.tar.gz",
+					"sha256": "e8e68959d7282ca147360fc9644ada9bd161bab781bab14d33b8999a95182781"
 				}
 			]
-		},	
+		},
 		{
 			"name": 		"LAPACK",
 			"buildsystem":	"cmake",
@@ -85,8 +94,8 @@
 					"type":   "archive",
 					"sha256": "8ac5dd57982bfd7d5f0ee384499d62f3e0bb35b5f1660feb368545f1186371fc",
 					"url":    "https://static.jasp-stats.org/RPkgs/JAGS-4.3.0.tar.gz"
-				} ]	
-		},	
+				} ]
+		},
 		"RPackages.json",
 		{
 			"name": "readstat",
@@ -100,12 +109,15 @@
 		{
 			"name": 		"jasp",
 			"buildsystem": 	"qmake",
+			"config-opts": [
+                "QMAKE_INCDIR+=/app/include/QtWebEngine"
+			],
 			"sources":
 			[
 				{
 					"type":		"git",
-					"tag":		"v0.13.0.0_rc0_flathub_fix0",
-					"url": 		"https://github.com/jasp-stats/jasp-desktop"
+					"commit":	"0046fa53f209a50577f25b439be74994844f2bff",
+					"url": 		"https://github.com/vandenman/jasp-desktop"
 				}
 			]
 		}

diff --git a/Tools/flatpak/setup-rpkgs/R/Rprofile.R b/Tools/flatpak/setup-rpkgs/R/Rprofile.R
@@ -10,7 +10,7 @@
 }
 
 stripUrl <- function(url) {
-  tolower(gsub("/+", "_", strsplit(url, "repos/", fixed = TRUE)[[1L]][2]))
+  gsub("/+", "_", strsplit(url, "repos/", fixed = TRUE)[[1L]][2])
 }
 
 download_override_flatpak <- function(url, destfile, mode = "wb", quiet = FALSE, headers = NULL) {

diff --git a/Tools/flatpak/setup-rpkgs/R/flatpakGeneratePkgsList.R b/Tools/flatpak/setup-rpkgs/R/flatpakGeneratePkgsList.R
@@ -1,45 +1,70 @@
-# This file needs internet!
-
-# TODO: look at https://github.com/andrie/miniCRAN/issues/50#issuecomment-374624319
-# it would be nice to be able to outsource this to miniCRAN which has a method for adding local pkgs to a repository
-# the only downside is that it looks like miniCRAN is not actively maintained anymore.
-
-# when running this file on a new computer, adjust these paths
-jaspDir    <- "~/github/jasp-desktop"               # local clone of https://github.com/jasp-stats/jasp-desktop
-flatpakDir <- "~/github/flatpak/org.jaspstats.JASP" # local clone of https://github.com/flathub/org.jaspstats.JASP
-
-source("R/functions.R")
-
-# you probably want to set a GITHUB_PAT because otherwise you WILL get rate-limited by GitHub.
-# Sys.setenv("GITHUB_PAT" = ...)
+# Since everybody has a different setup, it might be that you need to adjust some (minor) things
+# when running this on a new PC. The things you may need to adjust can be found by searching for
+# the comment `| HERE |`.
+#
+# Some other remarks:
+#
+# - you MUST set a GITHUB_PAT, otherwise you'll get an error. Do not commit that to this file because then GitHub will invalidate it.
+# - you can call this from cmd via Rscript R/flatpakGeneratePkgsList.R. (in Tools/flatpak/setup-rpkgs)
+
+# TODO:
+#
+# - look at https://github.com/andrie/miniCRAN/issues/50#issuecomment-374624319
+#   it would be nice to be able to outsource this to miniCRAN which has a method for adding local pkgs to a repository
+#   the only downside is that it looks like miniCRAN is not actively maintained anymore.
+#
+# - there is some rudimentary support for caching downloaded CRAN pkgs.
+#   this should be extended to GitHub packages as well, but it doesn't work because I delete all old files related to a particular
+#   github pkg if renv alreadyd downloaded it. So now we're downloading jaspBase 20 times.
+
+expectedDirs <- c("R", "renv")
+if (!all(expectedDirs %in% list.dirs(getwd(), recursive = FALSE, full.names = FALSE)))
+  stop("Incorrect working directory! Expected these directories at the top level: ", paste(expectedDirs, collapse = ", "))
+
+# | HERE | you may want to adjust the paths when running this file on a new computer
+jaspDir    <- normalizePath(file.path(getwd(), 	"..", "..", ".."))  		# local clone of https://github.com/jasp-stats/jasp-desktop
+flatpakDir <- normalizePath(file.path(jaspDir, "..", "org.jaspstats.JASP")) # local clone of https://github.com/flathub/org.jaspstats.JASP
+
+renvProject <- file.path(jaspDir, "Tools", "flatpak", "setup-rpkgs")
+if (!identical(renv::project(), renvProject))
+  renv::activate(renvProject)
+
+if (!identical(renv::project(), renvProject))
+  stop("Failed to set renv project")
+
+source(file.path("R", "functions.R"))
+source(file.path("R", "validators.R"))
+
+validateSetup(jaspDir, flatpakDir)
 
 options(repos = list(repos = c(CRAN = "https://cran.rstudio.com")))
 
-dirs <- setupJaspDirs("flatpak_folder")
 # NOTE: if you change the flatpak_dir anywhere you must also change it in the flatpak builder script!
+dirs <- setupJaspDirs("flatpak_folder")
 
 Sys.setenv("RENV_PATHS_CACHE" = dirs["renv-cache"])
 Sys.setenv("RENV_PATHS_ROOT"  = dirs["renv-root"])
 
-# use the default branch of all modules -- always the latest version
-jaspModules <- paste0("jasp-stats/", Filter(function(x) startsWith(x, "jasp"), dir(file.path(jaspDir, "Modules"))))
-
-# this uses the local versions -- but modules that are dependencies are still retrieved from github
-# isJaspModule <- function(path) file.exists(file.path(path, "DESCRIPTION")) && file.exists(file.path(path, "inst", "Description.qml"))
-# jaspModules <- Filter(isJaspModule, list.dirs("~/github/jasp-desktop/Modules", recursive = FALSE))
+# this version uses the default branch of all modules -- always the latest version
+# jaspModules <- paste0("jasp-stats/", Filter(function(x) startsWith(x, "jasp"), dir(file.path(jaspDir, "Modules"))))
 
+# this version uses the local checked-out versions -- but modules that are dependencies are still retrieved from github
+isJaspModule <- function(path) file.exists(file.path(path, "DESCRIPTION")) && file.exists(file.path(path, "inst", "Description.qml"))
+jaspModules <- Filter(isJaspModule, list.dirs(file.path(jaspDir, "Modules"), recursive = FALSE))
 names(jaspModules) <- basename(jaspModules)
 
-moduleEnvironments <- getModuleEnvironments(jaspModules)
-saveRDS(moduleEnvironments, file = file.path(dirs["module-environments"], "module-environments.rds"))
-# moduleEnvironments <- readRDS(file.path(dirs["module-environments"], "module-environments.rds"))
-# names(moduleEnvironments[[1]]$records)[1:5]
+# | HERE | you can add modules to exclude
+jaspModules <- jaspModules[setdiff(names(jaspModules), c("jaspCircular"))]
+
+getModuleEnvironments(jaspModules)
+# system("beep_finished.sh")
+
+moveMissingTarBalls(dirs)
 
 installRecommendedPackages(dirs)
 
 cleanupBigPackages(dirs)
 
-# downloadFakeV8(dirs)
 updateV8Rpackage(dirs)
 
 createLocalPackageRepository(dirs)
@@ -50,15 +75,17 @@ createLocalPackageRepository(dirs)
 # all(tb == 1)
 # tb[tb != 1] # these packages appear more than once
 
-# downloadV8(dirs)
+downloadV8ifneeded()
 copyV8Lib(dirs)
 copyRfiles(dirs)
 
+validateFlatpakFolder(dirs)
+
 # debugonce(createTarArchive)
 info <- createTarArchive(dirs, jaspDir, verbose = FALSE, compression = "best")
 
-# update Rpackages.json & install build flatpak
-writeRpkgsJson(file.path(flatpakDir, "RPackages.json"), info)
+# | HERE | you may wish to flip the local flag to adjust the script so it uses the local archive
+writeRpkgsJson(file.path(flatpakDir, "RPackages.json"), info, local = TRUE)
 
 # IF you have ssh setup this will upload the tar.gz to static-jasp. It's nicer to do this via a terminal because there you see a progress bar
-uploadTarArchive(info["tar-file"])
+# uploadTarArchive(info["tar-file"])
diff --git a/Tools/flatpak/setup-rpkgs/R/flatpakRegenerateRenvStructure.R b/Tools/flatpak/setup-rpkgs/R/flatpakRegenerateRenvStructure.R
@@ -46,8 +46,13 @@ dirV8 <- file.path(dirLib64, "v8")
 # This must be an absolute path, since installation is staged
 if (runningLocally) dirV8 <- normalizePath(dirV8)
 
+prettyCat(dir())
+prettyCat(dir(dirApp))
+
+dirLibGit2 <- file.path(dirApp, "libgit2")
 configureVars <- c(
-  V8 = sprintf("INCLUDE_DIR=%1$s/include LIB_DIR=%1$s/lib", dirV8)
+  V8   = sprintf("INCLUDE_DIR=%1$s/include LIB_DIR=%1$s/lib", dirV8),
+  gert = sprintf("INCLUDE_DIR=%1$s/include LIB_DIR=%1$s", dirLibGit2)
 )
 options(configure.vars = configureVars)
 prettyCat(configureVars)
@@ -57,8 +62,10 @@ libArch <- system("uname -m", intern = TRUE)
 Sys.setenv("LIB_ARCH" = if (identical(libArch, "x86_64")) "x64" else "aarch64")
 prettyCat(Sys.getenv("LIB_ARCH"))
 
-# install V8 here so later it only needs to be retrieved from the cache
-renv::install("V8")
+# install V8 and gert here so later they only need to be retrieved from the cache
+availablePkgs <- available.packages()
+toInstall <- intersect(c("V8", "gert"), availablePkgs[, "Package"])
+renv::install(toInstall)
 
 installJaspStats(c("jaspBase", "jaspGraphs"), dirs)