Merge pull request #2 from cyclic-software/stdio-file-passing

Stdio file passing

.github/workflows/release.yaml

@@ -7,6 +7,8 @@ on:
 env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: us-east-2
+  BUCKET: cyclic-sh-s3fs-test-bucket-us-east-2
   CYCLIC_DB: db-sdkCyclicDB
 
 jobs:

.github/workflows/run_tests.yaml

@@ -0,0 +1,25 @@
+name: integration tests
+
+on:
+  pull_request:
+    branches:
+      - main
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: us-east-2
+  BUCKET: cyclic-sh-s3fs-test-bucket-us-east-2
+  CYCLIC_DB: db-sdkCyclicDB
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '16.x'
+      - run: npm ci
+      - run: npm run test
+

env.sample

@@ -0,0 +1,3 @@
+# Exists in account: 758562997317 (dev-202101)
+BUCKET=cyclic-sh-s3fs-test-bucket-us-east-2
+AWS_REGION=us-east-2

package.json

@@ -5,7 +5,7 @@
   "type": "commonjs",
   "main": "src/index.js",
   "scripts": {
-    "test": "jest",
+    "test": "env-cmd --silent jest",
     "prepare": "husky install"
   },
   "keywords": [
@@ -31,6 +31,7 @@
   "devDependencies": {
     "@commitlint/cli": "^17.3.0",
     "@commitlint/config-conventional": "^17.3.0",
+    "env-cmd": "^10.1.0",
     "husky": "^8.0.2",
     "jest": "^29.3.1",
     "semantic-release": "^19.0.5"

src/index.js

@@ -32,22 +32,21 @@ class CyclicS3FS {
     return obj
   }
 
-  async writeFile(fileName, data, options) {
+  async writeFile(fileName, data, options={}) {
     const cmd = new PutObjectCommand({
         Bucket: this.bucket,
         Key: fileName,
         Body: data
     })
 
-    let result = await this.s3.send(cmd)
-    return result
+    await this.s3.send(cmd)
   }
 
   readFileSync(fileName) {
     return sync_interface.runSync(this,'readFile',[fileName])
   }
 
-  writeFileSync(fileName, data, options) {
+  writeFileSync(fileName, data, options={}) {
     return sync_interface.runSync(this,'writeFile',[fileName, data, options])
   }
 

src/sync_interface.js

@@ -4,18 +4,20 @@ const childProcess = require('child_process')
 const v8 = require('v8')
 const s3fs = require("./index.js")
 const HUNDRED_MEGABYTES = 1000 * 1000 * 100;
+var fs = require("fs");
 
 const runSync =  function(client, method, args){
     const bucket = client.bucket
     const config = JSON.stringify(client.config)
-    const args_serialized = v8.serialize(args).toString('hex');
+    const args_serialized = v8.serialize(args)
     const {error: subprocessError, stdout, stderr} = childProcess.spawnSync(
         `node`, [`${path.resolve(__dirname,'./sync_interface.js')}`,
                 bucket,
                 config,
                 method,
-                args_serialized
+                // args_serialized
             ], {
+                input: Buffer.from(args_serialized),
                 maxBuffer: HUNDRED_MEGABYTES,
                 env: {
                     ...process.env,
@@ -34,10 +36,8 @@ const runSync =  function(client, method, args){
       }
 
 
-      let result = stdout?.toString()
-      if(result){
-        const r = v8.deserialize(Buffer.from(result,'hex'));
-        return r
+      if(stdout){
+        return stdout
       }
 }
 
@@ -47,20 +47,26 @@ module.exports = {
     runSync,
 }
 const run = async function(bucket, config, method, args){
-
     const fs = new s3fs(bucket, config)
     let result = await fs[method](...args)
-
-    const serialized = v8.serialize(result).toString('hex');
-    process.stdout.write(serialized);
+    if(result){
+        process.stdout.write(result);
+    }
 }
 
 if (require.main === module) {
+
     let _argv = process.argv.slice(2,)
     let bucket = _argv[0]
     let config = JSON.parse(_argv[1])
     let method = _argv[2]
-    let args = _argv[3]
-    args = v8.deserialize(Buffer.from(args,'hex'));
-    run(bucket, config, method, args)
+    var buf = '';
+
+    process.stdin.on('data', function(d) {
+        buf += d;
+      }).on('end', function() {
+        let args = v8.deserialize(Buffer.from(buf,'binary'));
+        run(bucket, config, method, args)
+      }).setEncoding('binary');
+
 }