[WIP] Download annotated notebooks

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## rmapi master
 
+* Added geta and mgeta command to read annotated notebooks
+
 * Increased http timeout to 5 minutes to enable upload of larger files
 
 * Add user-agent header to be a good reMarkable citezen

README.md

@@ -27,6 +27,15 @@ Install and build the project:
 
 `go get -u github.com/juruen/rmapi`
 
+Also the following tools are needed to read annotated notebooks:
+ - convert (Note: Policy must be set to read|write for pdfs: https://stackoverflow.com/questions/42928765/convertnot-authorized-aaaa-error-constitute-c-readimage-453)
+ - rsvg-convert
+ - pdf270
+ - pdftk
+ - pdfunite
+ - python3
+
+
 ## Binary
 
 You can download an already built version for either Linux or OSX from [releases](https://github.com/juruen/rmapi/releases).
@@ -88,6 +97,11 @@ mput /Papers
 
 Use `get path_to_file` to download a file from the cloud to your local computer.
 
+## Download a file with annotations
+Use `geta path_to_file` to download a file from the cloud to your local computer.
+If there exists a template folder under `%HOME/Templates/Remarkable`, then those 
+are used for notebooks. Otherwise the background is white.
+
 ## Recursively download directories and files
 
 Use `mget path_to_dir` to recursively download all the files in that directory.
@@ -98,6 +112,17 @@ E.g: download all the files
 mget .
 ```
 
+## Recursively download directories and files with annotations
+
+Use `mgeta path_to_dir` to recursively download all the annotated files in that directory.
+
+E.g: download all the files
+
+```
+mgeta .
+```
+
+
 ## Create a directoy
 
 Use `mkdir path_to_new_dir` to create a new directory
@@ -148,3 +173,8 @@ If you want to give it a go, you can run:
 ```bash
 rmapi --fuse-mount mount_point
 ```
+
+
+# Thanks to
+[1] rmapi, https://github.com/juruen/rmapi <br />
+[1] rM2svg, https://github.com/reHackable/maxio <br />

shell/geta.go

@@ -0,0 +1,185 @@
+package shell
+
+import (
+	"errors"
+	"fmt"
+	"time"
+	"os/exec"
+	"strings"
+	"archive/zip"
+	"os"
+	"io"
+	"github.com/abiosoft/ishell"
+	"path/filepath"
+	"github.com/juruen/rmapi/model"
+)
+
+
+func getaCmd(ctx *ShellCtxt) *ishell.Cmd {
+	return &ishell.Cmd{
+		Name:      "geta",
+		Help:      "copy remote file to local with annotations. Optional [svg, pdf]",
+		Completer: createEntryCompleter(ctx),
+		Func: func(c *ishell.Context) {
+			// Parse cmd args
+			if len(c.Args) == 0 {
+				c.Err(errors.New("missing source file"))
+				return
+			}
+			srcName := c.Args[0]
+
+			// Download document as zip
+			node, err := ctx.api.Filetree.NodeByPath(srcName, ctx.node)
+			if err != nil || node.IsDirectory() {
+				c.Err(errors.New("file doesn't exist"))
+				return
+			}
+
+			err = getAnnotatedDocument(ctx, node, "")
+			if err != nil {
+				c.Err(err)
+				return
+			}
+		},
+	}
+}
+
+
+func getAnnotatedDocument(ctx *ShellCtxt, node *model.Node, path string) error {
+	zipFile := fmt.Sprintf(".%s.zip", node.Name())
+
+	// Set output name and output file name
+	output := node.Name()
+	if(path != ""){
+		output = fmt.Sprintf("%s/%s", path, node.Name())
+	}
+	outputFileName := fmt.Sprintf("%s.pdf", output)
+
+	// Parse last modified time of document on api
+	modifiedClientTime, err := time.Parse(time.RFC3339Nano, node.Document.ModifiedClient)
+	if err != nil {
+		// If we could not parse the time correctly, we still continue 
+		// with the execution such that the pdf is downloaded...
+		fmt.Println(err)
+		fmt.Println("(Warning) Could not parse modified time. Overwrite existing file.")
+		modifiedClientTime = time.Now().Local()
+	}
+
+	// If document has not changed since last update skip pdf convertion
+	outputFile, err := os.Stat(outputFileName)
+	if !os.IsNotExist(err) {
+		outputFileModTime := outputFile.ModTime()
+		if(outputFileModTime.Equal(modifiedClientTime)){
+			fmt.Println("Nothing changed since last download. Skip. ")
+			os.Remove(zipFile)
+			return nil
+		}
+	}
+
+	// Download document if content has changed
+	err = ctx.api.FetchDocument(node.Document.ID, zipFile)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Failed to download file %s with %s", node.Name(), err.Error()))
+	}
+
+	// Unzip document
+	tmpFolder := fmt.Sprintf(".%s", node.Document.ID)
+	_, err = unzip(zipFile, tmpFolder)
+	if err != nil {
+		os.Remove(zipFile)
+		return err
+	}
+
+	// Convert to pdf
+	exportPdf := os.Getenv("GOPATH") + "/src/github.com/juruen/rmapi/tools/exportAnnotatedPdf"
+	rM2svg := os.Getenv("GOPATH") + "/src/github.com/juruen/rmapi/tools/rM2svg"
+	_, err = exec.Command(
+		"/bin/bash", 
+		exportPdf, 
+		tmpFolder,
+		node.Document.ID, 
+		output, 
+		rM2svg).CombinedOutput()
+
+	if err != nil {
+	   fmt.Println(err)
+	}
+
+	if err != nil {
+		os.Remove(zipFile)
+		os.RemoveAll(tmpFolder)
+		return err
+	}
+
+	// Set creation time
+	err = os.Chtimes(outputFileName, modifiedClientTime, modifiedClientTime)
+	if err != nil {
+		fmt.Println("(Warning) Could not set modified time of pdf file.")
+	}
+
+	// Cleanup
+	os.Remove(zipFile)
+	os.RemoveAll(tmpFolder)
+	fmt.Println("Ok.")
+	return nil
+}
+
+// From https://golangcode.com/unzip-files-in-go/
+func unzip(src string, dest string) ([]string, error) {
+
+    var filenames []string
+
+    r, err := zip.OpenReader(src)
+    if err != nil {
+        return filenames, err
+    }
+    defer r.Close()
+
+    for _, f := range r.File {
+
+        rc, err := f.Open()
+        if err != nil {
+            return filenames, err
+        }
+        defer rc.Close()
+
+        // Store filename/path for returning and using later on
+        fpath := filepath.Join(dest, f.Name)
+
+        // Check for ZipSlip. More Info: http://bit.ly/2MsjAWE
+        if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) {
+            return filenames, fmt.Errorf("%s: illegal file path", fpath)
+        }
+
+        filenames = append(filenames, fpath)
+
+        if f.FileInfo().IsDir() {
+
+            // Make Folder
+            os.MkdirAll(fpath, os.ModePerm)
+
+        } else {
+
+            // Make File
+            if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {
+                return filenames, err
+            }
+
+            outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+            if err != nil {
+                return filenames, err
+            }
+
+            _, err = io.Copy(outFile, rc)
+
+            // Close the file without defer to close before next iteration of loop
+            outFile.Close()
+
+            if err != nil {
+                return filenames, err
+            }
+
+        }
+    }
+    return filenames, nil
+}

shell/mgeta.go

@@ -0,0 +1,66 @@
+package shell
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/abiosoft/ishell"
+	"github.com/juruen/rmapi/filetree"
+	"github.com/juruen/rmapi/model"
+)
+
+func mgetaCmd(ctx *ShellCtxt) *ishell.Cmd {
+	return &ishell.Cmd{
+		Name:      "mgeta",
+		Help:      "recursively copy remote directory to local with annotations",
+		Completer: createDirCompleter(ctx),
+		Func: func(c *ishell.Context) {
+			if len(c.Args) == 0 {
+				c.Err(errors.New(("missing source dir")))
+				return
+			}
+
+			srcName := c.Args[0]
+
+			node, err := ctx.api.Filetree.NodeByPath(srcName, ctx.node)
+
+			if err != nil || node.IsFile() {
+				c.Err(errors.New("directory doesn't exist"))
+				return
+			}
+
+			visitor := filetree.FileTreeVistor{
+				func(currentNode *model.Node, currentPath []string) bool {
+					idxDir := 0
+					if srcName == "." && len(currentPath) > 0 {
+						idxDir = 1
+					}
+
+					dst := "./" + filetree.BuildPath(currentPath[idxDir:], currentNode.Name())
+					dir := path.Dir(dst)
+					os.MkdirAll(dir, 0766)
+
+					if currentNode.IsDirectory() {
+						return filetree.ContinueVisiting
+					}
+
+					c.Printf("downloading [%s]...", dst)
+
+					err = getAnnotatedDocument(ctx, currentNode, fmt.Sprintf("%s", dir))
+
+					if err == nil {
+						return filetree.ContinueVisiting
+					}
+
+					c.Err(errors.New(fmt.Sprintf("Failed to downlaod file %s", currentNode.Name())))
+
+					return filetree.ContinueVisiting
+				},
+			}
+
+			filetree.WalkTree(node, visitor)
+		},
+	}
+}

shell/shell.go

@@ -41,6 +41,8 @@ func RunShell(apiCtx *api.ApiCtx) error {
 	shell.AddCmd(cdCmd(ctx))
 	shell.AddCmd(getCmd(ctx))
 	shell.AddCmd(mgetCmd(ctx))
+	shell.AddCmd(getaCmd(ctx))
+	shell.AddCmd(mgetaCmd(ctx))
 	shell.AddCmd(mkdirCmd(ctx))
 	shell.AddCmd(rmCmd(ctx))
 	shell.AddCmd(mvCmd(ctx))