support arbitrary amounts of heap memory

src/ps5/elfldr.c

@@ -412,10 +412,49 @@ elfldr_prepare_exec(pid_t pid, uint8_t *elf) {
 }
 
 
+int
+elfldr_set_heap_size(pid_t pid, ssize_t size) {
+  intptr_t sceLibcHeapSize;
+  intptr_t sceLibcParam;
+  intptr_t sceProcParam;
+  intptr_t Need_sceLibc;
+
+  if(!(sceProcParam=pt_sceKernelGetProcParam(pid))) {
+    pt_perror(pid, "pt_sceKernelGetProcParam");
+    return -1;
+  }
+
+  if(mdbg_copyout(pid, sceProcParam+56, &sceLibcParam,
+		  sizeof(sceLibcParam))) {
+    perror("mdbg_copyout");
+    return -1;
+  }
+
+  if(mdbg_copyout(pid, sceLibcParam+16, &sceLibcHeapSize,
+		  sizeof(sceLibcHeapSize))) {
+    perror("mdbg_copyout");
+    return -1;
+  }
+
+  if(mdbg_setlong(pid, sceLibcHeapSize, size)) {
+    perror("mdbg_setlong");
+    return -1;
+  }
+
+  if(size != -1) {
+    return 0;
+  }
+
+  if(mdbg_copyout(pid, sceLibcParam+72, &Need_sceLibc,
+		  sizeof(Need_sceLibc))) {
+    perror("mdbg_copyout");
+    return -1;
+  }
+
+  return mdbg_setlong(pid, sceLibcParam+32, Need_sceLibc);
+}
+
 
-/**
- * Set the current working directory.
- **/
 int
 elfldr_set_cwd(pid_t pid, const char* cwd) {
   intptr_t buf;

src/ps5/elfldr.h

@@ -42,4 +42,13 @@ int elfldr_set_procname(pid_t pid, const char* name);
 int elfldr_set_stdio(pid_t pid, int stdio);
 
 
+/**
+ * Set the current working directory.
+ **/
 int elfldr_set_cwd(pid_t pid, const char* cwd);
+
+
+/**
+ * Set the heap size for libc.
+ **/
+int elfldr_set_heap_size(pid_t pid, ssize_t size);

src/ps5/hbldr.c

@@ -15,6 +15,7 @@ along with this program; see the file COPYING. If not, see
 <http://www.gnu.org/licenses/>.  */
 
 #include <fcntl.h>
+#include <pthread.h>
 #include <signal.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -71,6 +72,10 @@ typedef struct app_launch_ctx {
   uint32_t app_opt;
   uint64_t crash_report;
   uint32_t check_flag;
+
+  // End of SCE fields, the following fields are just used to
+  // pass arguments to bigapp_launch_thread().
+  char **argv;
 } app_launch_ctx_t;
 
 
@@ -250,56 +255,78 @@ find_pid(const char* name) {
 }
 
 
+static void*
+bigapp_launch_thread(void* args) {
+  app_launch_ctx_t *ctx  = (app_launch_ctx_t *)args;
+
+  sceSystemServiceLaunchApp("FAKE00000", ctx->argv, ctx);
+
+  return 0;
+}
+
+
 static pid_t
 bigapp_launch(uint32_t user_id, char** argv) {
-  app_launch_ctx_t ctx = {.user_id = user_id};
-  struct kevent evt;
-  pid_t pid = -1;
-  int kq;
+  app_launch_ctx_t ctx = {.user_id = user_id, .argv = argv};
+  pthread_t trd;
+  pid_t parent;
+  pid_t child;
 
-  if((pid=find_pid("SceSysCore.elf")) < 0) {
+  if((parent=find_pid("SceSysCore.elf")) < 0) {
     puts("SceSysCore.elf is not running?");
     return -1;
   }
 
-  if((kq=kqueue()) < 0) {
-    perror("kqueue");
+  if(pt_attach(parent) < 0) {
+    perror("pt_attach");
     return -1;
   }
 
-  EV_SET(&evt, pid, EVFILT_PROC, EV_ADD, NOTE_FORK | NOTE_TRACK, 0, NULL);
-  if(kevent(kq, &evt, 1, NULL, 0, NULL) < 0) {
-    perror("kevent");
-    close(kq);
+  if(pt_follow_fork(parent) < 0) {
+    perror("pt_follow_fork");
+    pt_detach(parent, 0);
     return -1;
   }
 
-  if(sceSystemServiceLaunchApp("FAKE00000", argv, &ctx) < 0) {
-    perror("sceSystemServiceLaunchApp");
-    close(kq);
+  if(pt_continue(parent, SIGCONT) < 0) {
+    perror("pt_continue");
+    pt_detach(parent, 0);
     return -1;
   }
 
-  while(1) {
-    if(kevent(kq, NULL, 0, &evt, 1, NULL) < 0) {
-      perror("kevent");
-      break;
-    }
+  pthread_create(&trd, 0, &bigapp_launch_thread, &ctx);
+  if((child=pt_await_child(parent)) < 0) {
+    perror("pt_await_child");
+    pt_detach(parent, 0);
+    return -1;
+  }
 
-    if(evt.fflags & NOTE_CHILD) {
-      pid = evt.ident;
-      break;
-    }
+  if(pt_detach(parent, 0) < 0) {
+    perror("pt_detach");
+    return -1;
+  }
+
+  if(pt_follow_exec(child) < 0) {
+    perror("pt_follow_exec");
+    pt_detach(child, SIGKILL);
+    return -1;
+  }
+
+  if(pt_continue(child, SIGCONT) < 0) {
+    perror("pt_continue");
+    pt_detach(child, SIGKILL);
+    return -1;
   }
 
-  if(kill(pid, SIGSTOP) < 0) {
-    perror("kill");
+  if(pt_await_exec(child)) {
+    perror("pt_await_exec");
+    pt_detach(child, SIGKILL);
     return -1;
   }
 
-  close(kq);
+  pthread_join(trd, 0);
 
-  return pid;
+  return child;
 }
 
 
@@ -349,15 +376,14 @@ bigapp_replace(pid_t pid, uint8_t* elf, const char* progname, int stdio,
   uint8_t orginstr;
   char* cwd;
 
-  if(!(cwd=getenv("PWD"))) {
-    cwd = getcwd(buf, sizeof(buf));
+  // Let the kernel assign process parameters accessed via sceKernelGetProcParam()
+  if(pt_syscall(pid, 599)) {
+    puts("sys_dynlib_process_needed_and_relocate failed");
+    //return -1;
   }
 
-  if(pt_attach(pid) < 0) {
-    perror("pt_attach");
-    kill(pid, SIGKILL);
-    return -1;
-  }
+  // Allow libc to allocate arbitrary amount of memory.
+  elfldr_set_heap_size(pid, -1);
 
   if(!(brkpoint=kernel_dynlib_entry_addr(pid, 0))) {
     puts("kernel_dynlib_entry_addr failed");
@@ -387,6 +413,10 @@ bigapp_replace(pid_t pid, uint8_t* elf, const char* progname, int stdio,
     return -1;
   }
 
+  if(!(cwd=getenv("PWD"))) {
+    cwd = getcwd(buf, sizeof(buf));
+  }
+
   bigapp_set_argv0(pid, progname);
   elfldr_set_procname(pid, basename(progname));
   elfldr_set_environ(pid, envp);

src/ps5/pt.c

@@ -108,6 +108,71 @@ pt_detach(pid_t pid, int sig) {
   return 0;
 }
 
+int
+pt_follow_fork(pid_t pid) {
+  if(sys_ptrace(PT_FOLLOW_FORK, pid, NULL, 1) == -1) {
+    return -1;
+  }
+
+  if(sys_ptrace(PT_LWP_EVENTS, pid, NULL, 1) == -1) {
+    return -1;
+  }
+
+  return 0;
+}
+
+
+int
+pt_follow_exec(pid_t pid) {
+  if(sys_ptrace(PT_LWP_EVENTS, pid, NULL, 1) == -1) {
+    return -1;
+  }
+
+  return 0;
+}
+
+
+pid_t
+pt_await_child(pid_t pid) {
+  struct ptrace_lwpinfo lwpinfo;
+
+  memset(&lwpinfo, 0, sizeof(lwpinfo));
+  while(!(lwpinfo.pl_flags & PL_FLAG_FORKED)) {
+    if(waitpid(pid, NULL, 0) == -1) {
+      return -1;
+    }
+
+    if(sys_ptrace(PT_LWPINFO, pid, (caddr_t)&lwpinfo, sizeof(lwpinfo)) == -1) {
+      return -1;
+    }
+  }
+
+  if(waitpid(lwpinfo.pl_child_pid, NULL, 0) == -1) {
+    return -1;
+  }
+
+  return lwpinfo.pl_child_pid;
+}
+
+
+int
+pt_await_exec(pid_t pid) {
+  struct ptrace_lwpinfo lwpinfo;
+
+  memset(&lwpinfo, 0, sizeof(lwpinfo));
+  while(!(lwpinfo.pl_flags & PL_FLAG_EXEC)) {
+    if(waitpid(pid, NULL, 0) == -1) {
+      return -1;
+    }
+
+    if(sys_ptrace(PT_LWPINFO, pid, (caddr_t)&lwpinfo, sizeof(lwpinfo)) == -1) {
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
 
 int
 pt_step(int pid) {

src/ps5/pt.h

@@ -26,6 +26,12 @@ int pt_detach(pid_t pid, int sig);
 int pt_step(pid_t pid);
 int pt_continue(pid_t pid, int sig);
 
+int pt_follow_fork(pid_t pid);
+int pt_follow_exec(pid_t pid);
+
+pid_t pt_await_child(pid_t pid);
+int pt_await_exec(pid_t pid);
+
 int pt_getregs(pid_t pid, struct reg *r);
 int pt_setregs(pid_t pid, const struct reg *r);