Merge pull request #127 from Human-Connection/100_2_download_uploads_…

…and_more

Follow up on data import

.dockerignore

@@ -18,3 +18,6 @@ dist/
 
 db-migration-worker/
 neo4j/
+
+public/uploads/*
+!.gitkeep

.gitignore

@@ -6,3 +6,5 @@ yarn-error.log
 dist/*
 coverage.lcov
 .nyc_output/
+public/uploads/*
+!.gitkeep

db-migration-worker/Dockerfile

@@ -1,7 +1,7 @@
 FROM mongo:4
 
 RUN apt-get update \
- && apt-get -y install --no-install-recommends openssh-client \
+ && apt-get -y install --no-install-recommends openssh-client rsync \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 COPY .ssh /root/.ssh/

db-migration-worker/import.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB"
+for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" "UPLOADS_DIRECTORY"
 do
   if [[ -z "${!var}" ]]; then
     echo "${var} is undefined"
@@ -14,6 +14,7 @@ echo "MONGODB_USERNAME         ${MONGODB_USERNAME}"
 echo "MONGODB_PASSWORD         ${MONGODB_PASSWORD}"
 echo "MONGODB_DATABASE         ${MONGODB_DATABASE}"
 echo "MONGODB_AUTH_DB          ${MONGODB_AUTH_DB}"
+echo "UPLOADS_DIRECTORY        ${UPLOADS_DIRECTORY}"
 echo "-------------------------------------------------"
 
 mongo ${MONGODB_DATABASE} --eval "db.dropDatabase();"
@@ -24,6 +25,8 @@ mongodump --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONG
 ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST}
 ssh -S my-ctrl-socket -O exit  -l ${SSH_USERNAME} ${SSH_HOST}
 
+rsync --archive --update --verbose ${SSH_USERNAME}@${SSH_HOST}:${UPLOADS_DIRECTORY}/* /uploads/
+
 for collection in "categories" "badges" "users" "contributions" "comments" "follows" "shouts"
 do
   mongoexport --db ${MONGODB_DATABASE} --collection $collection --out "/mongo-export/$collection.json"

docker-compose.override.yml

@@ -9,6 +9,7 @@ services:
     volumes:
       - .:/nitro-backend
       - /nitro-backend/node_modules
+      - uploads:/nitro-backend/public/uploads
     command: yarn run dev
   neo4j:
     volumes:
@@ -24,6 +25,7 @@ services:
       context: db-migration-worker
     volumes:
       - mongo-export:/mongo-export
+      - uploads:/uploads
       - ./db-migration-worker/.ssh/:/root/.ssh/
     networks:
       - hc-network
@@ -34,7 +36,9 @@ services:
       - "MONGODB_PASSWORD=${MONGODB_PASSWORD}"
       - "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}"
       - "MONGODB_DATABASE=${MONGODB_DATABASE}"
+      - "UPLOADS_DIRECTORY=${UPLOADS_DIRECTORY}"
     command: "--smallfiles --logpath=/dev/null"
 
 volumes:
   mongo-export:
+  uploads:

neo4j/import/badges.cql

@@ -0,0 +1,10 @@
+CALL apoc.load.json('file:/mongo-export/badges.json') YIELD value as badge
+MERGE(b:Badge {id: badge._id["$oid"]})
+ON CREATE SET
+b.key       = badge.key,
+b.type      = badge.type,
+b.icon      = badge.image.path,
+b.status    = badge.status,
+b.createdAt = badge.createdAt.`$date`,
+b.updatedAt = badge.updatedAt.`$date`
+;

neo4j/import/categories.cql

@@ -0,0 +1,89 @@
+CALL apoc.load.json('file:/mongo-export/categories.json') YIELD value as category
+MERGE(c:Category {id: category._id["$oid"]})
+ON CREATE SET
+c.name      = category.title,
+c.slug      = category.slug,
+c.icon      = category.icon,
+c.createdAt = category.createdAt.`$date`,
+c.updatedAt = category.updatedAt.`$date`
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-justforfun")
+SET c.icon = 'smile'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-luck")
+SET c.icon = 'heart-o'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-health")
+SET c.icon = 'medkit'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-environment")
+SET c.icon = 'tree'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-animal-justice")
+SET c.icon = 'paw'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-human-rights")
+SET c.icon = 'balance-scale'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-education")
+SET c.icon = 'graduation-cap'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-cooperation")
+SET c.icon = 'users'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-politics")
+SET c.icon = 'university'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-economy")
+SET c.icon = 'money'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-technology")
+SET c.icon = 'flash'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-internet")
+SET c.icon = 'mouse-pointer'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-art")
+SET c.icon = 'paint-brush'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-freedom-of-speech")
+SET c.icon = 'bullhorn'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-sustainability")
+SET c.icon = 'shopping-cart'
+;
+
+MATCH (c:Category)
+WHERE (c.icon = "categories-peace")
+SET c.icon = 'angellist'
+;

neo4j/import/follows.cql

@@ -0,0 +1,4 @@
+CALL apoc.load.json('file:/mongo-export/follows.json') YIELD value as follow
+MATCH (u1:User {id: follow.userId}), (u2:User {id: follow.foreignId})
+MERGE (u1)-[:FOLLOWS]->(u2)
+;

neo4j/import/import.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell
-for collection in "users" "contributions" "comments"
+for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments"
 do
   echo "Import ${collection}..." && cypher-shell < $SCRIPT_DIRECTORY/$collection.cql
 done

neo4j/import/shouts.cql

@@ -0,0 +1,4 @@
+CALL apoc.load.json('file:/mongo-export/shouts.json') YIELD value as shout
+MATCH (u:User {id: shout.userId}), (p:Post {id: shout.foreignId})
+MERGE (u)-[:SHOUTED]->(p)
+;

neo4j/import/todo

@@ -1,29 +1,2 @@
-CALL apoc.load.json('file:/mongo-export/categories.json') YIELD value as category
-MERGE(c:Category {id: category._id["$oid"]})
-ON CREATE SET  c.name  = category.title,
-    c.slug = category.slug,
-    c.icon = category.icon
 
 
-CALL apoc.load.json('file:/mongo-export/badges.json') YIELD value as badge
-MERGE(b:Badge {id: badge._id["$oid"]})
-ON CREATE SET  b.key   = badge.key,
-    b.type  = badge.type,
-    b.icon  = badge.image.path,
-    b.status  = badge.status
-
-
-
-
-
-
-CALL apoc.load.json('file:/mongo-export/follows.json') YIELD value as follow
-MATCH (u1:User {id: follow.userId}),
-   (u2:User {id: follow.foreignId})
-MERGE (u1)-[:FOLLOWS]->(u2)
-
-
-CALL apoc.load.json('file:/mongo-export/shouts.json') YIELD value as shout
-MATCH (u:User {id: shout.userId}),
-   (p:Post {id: shout.foreignId})
-MERGE (u)-[:SHOUTED]->(p)