Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deprecated: Add protobuf module as wasm module #2054

Open
wants to merge 38 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
eada60e
Add protobuf module wrapped into wasm module
DmitryAstafyev Jul 18, 2024
388bd45
Refactoring design of protocol scheme
DmitryAstafyev Jul 19, 2024
d50b923
Update unbound session
DmitryAstafyev Jul 22, 2024
e350b11
Remove unused code
DmitryAstafyev Jul 22, 2024
1912083
Fix typo of struct name
DmitryAstafyev Jul 22, 2024
bbd6407
Map protocol messages to TS
DmitryAstafyev Jul 23, 2024
cb0e31b
Fix callback event handeling
DmitryAstafyev Jul 23, 2024
ab325af
Fix stream (process) call
DmitryAstafyev Jul 23, 2024
962c0df
Update unbound session (jobs)
DmitryAstafyev Jul 23, 2024
b8cbd94
Update jobs controller on TS level
DmitryAstafyev Jul 24, 2024
f1ebb46
Refactoring code & simple performance test
DmitryAstafyev Jul 24, 2024
557ec00
Use buffer instead Vec<i32> for incoming from JS
DmitryAstafyev Jul 24, 2024
cfb77cf
Add support progress event
DmitryAstafyev Jul 25, 2024
8e1bc54
Update build script (add protocol package)
DmitryAstafyev Jul 25, 2024
2013079
Update environment checks & workflows (add protoc)
DmitryAstafyev Jul 25, 2024
88b7fbb
Switch to using SafeArray
DmitryAstafyev Sep 9, 2024
26c2810
Fix parsing issue
DmitryAstafyev Sep 10, 2024
8cfbd1e
Rebase to master (update jasmine runners)
DmitryAstafyev Sep 18, 2024
d8fc828
Cleanup
DmitryAstafyev Sep 19, 2024
b1206c0
Update a way to generate *.ts files based on protobuf scheme
DmitryAstafyev Sep 26, 2024
8a9039d
Update build script for protobuf
DmitryAstafyev Sep 27, 2024
75a9f2a
Github CI: use only latest LTS node
DmitryAstafyev Oct 18, 2024
06b741d
Cleanup CallbackEvent on ts-binding level
DmitryAstafyev Oct 23, 2024
c4af2e3
Cleanup tracker events
DmitryAstafyev Oct 23, 2024
627272d
Fix error decoding on ts-binding
DmitryAstafyev Oct 23, 2024
b0e0c74
Resolve rebase issues
DmitryAstafyev Nov 12, 2024
ba13ca8
Add test for CallbackEvent (all cases)
DmitryAstafyev Nov 12, 2024
7fe5f2b
Extend Jasmine tests with events and requests
DmitryAstafyev Nov 12, 2024
a8fdac2
Fix observe options test
DmitryAstafyev Nov 13, 2024
165627b
Add test for observe options proto-messages
DmitryAstafyev Nov 13, 2024
ab31027
Get rid of nested useless nested folder
DmitryAstafyev Nov 20, 2024
1484e42
Add bash generate script
DmitryAstafyev Nov 20, 2024
ddb0d06
Add readme.md for protocol description
DmitryAstafyev Nov 20, 2024
4fbbec2
Resolve conflicts
DmitryAstafyev Nov 20, 2024
91ea391
Resolve clippy issues
DmitryAstafyev Nov 20, 2024
a5a2f5e
Include compiled wasm protobuf module
DmitryAstafyev Nov 20, 2024
43d311d
Update workflows
DmitryAstafyev Nov 20, 2024
9f8c7e0
Include JS files of proto-wasm module
DmitryAstafyev Nov 21, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add support progress event
  • Loading branch information
DmitryAstafyev committed Nov 20, 2024
commit cfb77cff97b722c00fb2184909b11679f4d08d34
41 changes: 41 additions & 0 deletions application/apps/protocol/binding/wasm/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions application/apps/protocol/binding/wasm/src/msg/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@ pub mod error;
pub mod event;
pub mod grabbing;
pub mod observe;
pub mod progress;
pub mod sde;
17 changes: 17 additions & 0 deletions application/apps/protocol/binding/wasm/src/msg/progress.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
use crate::*;
use extend::extend;
use prost::Message;
use serde_wasm_bindgen::{from_value, to_value};
use wasm_bindgen::prelude::*;

#[extend(progress)]
pub struct Ticks;

#[extend(progress)]
pub struct LifecycleTransition;

#[extend(progress)]
pub struct Started;

#[extend(progress)]
pub struct TicksWithUuid;
62 changes: 43 additions & 19 deletions application/apps/rustcore/ts-bindings/spec/session.protocol.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,27 +59,51 @@ describe('Protocol', function () {
.then((session: Session) => {
// Set provider into debug mode
session.debug(true);
const MESSAGES_COUNT = 100000;
const meausere: { json: number; proto: number } = { json: 0, proto: 0 };
meausere.json = Date.now();
for (let i = MESSAGES_COUNT; i >= 0; i -= 1) {
const msg = session.getNativeSession().testGrabElsAsJson();
expect(msg instanceof Array).toBe(true);
const MESSAGES_COUNT = 100;
{
const meausere: { json: number; proto: number } = { json: 0, proto: 0 };
meausere.json = Date.now();
for (let i = MESSAGES_COUNT; i >= 0; i -= 1) {
const msg = session.getNativeSession().testGrabElsAsJson(false);
expect(msg instanceof Array).toBe(true);
}
meausere.json = Date.now() - meausere.json;
meausere.proto = Date.now();
for (let i = MESSAGES_COUNT; i >= 0; i -= 1) {
const msg = session.getNativeSession().testGrabElsAsProto(false);
expect(msg instanceof Array).toBe(true);
}
meausere.proto = Date.now() - meausere.proto;
console.log(
`Receiving messages count: ${MESSAGES_COUNT}\nJSON: ${
meausere.json
}ms (per msg ${(meausere.json / MESSAGES_COUNT).toFixed(2)});\nPROTO: ${
meausere.proto
}ms (per msg ${(meausere.proto / MESSAGES_COUNT).toFixed(2)})`,
);
}
meausere.json = Date.now() - meausere.json;
meausere.proto = Date.now();
for (let i = MESSAGES_COUNT; i >= 0; i -= 1) {
const msg = session.getNativeSession().testGrabElsAsProto();
expect(msg instanceof Array).toBe(true);
{
const meausere: { json: number; proto: number } = { json: 0, proto: 0 };
meausere.json = Date.now();
for (let i = MESSAGES_COUNT; i >= 0; i -= 1) {
const msg = session.getNativeSession().testGrabElsAsJson();
expect(msg instanceof Array).toBe(true);
}
meausere.json = Date.now() - meausere.json;
meausere.proto = Date.now();
for (let i = MESSAGES_COUNT; i >= 0; i -= 1) {
const msg = session.getNativeSession().testGrabElsAsProto();
expect(msg instanceof Array).toBe(true);
}
meausere.proto = Date.now() - meausere.proto;
console.log(
`Grabbing messages count: ${MESSAGES_COUNT}\nJSON: ${
meausere.json
}ms (per msg ${(meausere.json / MESSAGES_COUNT).toFixed(2)});\nPROTO: ${
meausere.proto
}ms (per msg ${(meausere.proto / MESSAGES_COUNT).toFixed(2)})`,
);
}
meausere.proto = Date.now() - meausere.proto;
console.log(
`Grabbing messages count: ${MESSAGES_COUNT}\nJSON: ${
meausere.json
}ms (per msg ${(meausere.json / MESSAGES_COUNT).toFixed(2)});\nPROTO: ${
meausere.proto
}ms (per msg ${(meausere.proto / MESSAGES_COUNT).toFixed(2)})`,
);
finish(session, done);
})
.catch((err: Error) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import { EErrorKind, EErrorSeverity } from '../provider/provider.errors';
import { IMapEntity, IMatchEntity, IValuesMinMaxMap } from 'platform/types/filter';
import { IAttachment } from 'platform/types/content';

import * as Types from '../protocol';

export interface IProgressState {
total: number;
count: number;
Expand Down Expand Up @@ -194,7 +196,7 @@ export class EventProvider extends Computation<
private readonly _convertors: ISessionEventsConvertors = {};

constructor(uuid: string) {
super(uuid);
super(uuid, Types.decodeCallbackEvent);
}

public getName(): string {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { Subject } from 'platform/env/subscription';
import { Computation } from '../provider/provider';

import * as Types from '../protocol';

export interface Job {
alias: string;
uuid: string;
Expand Down Expand Up @@ -74,7 +76,7 @@ export class EventProvider extends Computation<
private readonly _convertors = {};

constructor(uuid: string) {
super(uuid);
super(uuid, Types.decodeLifecycleTransition);
}

public getName(): string {
Expand Down
18 changes: 13 additions & 5 deletions application/apps/rustcore/ts-bindings/src/native/native.session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -204,10 +204,10 @@ export abstract class RustSession extends RustSessionRequiered {
public abstract triggerTrackerError(): Promise<void>;

// Used only for testing and debug
public abstract testGrabElsAsJson(): IGrabbedElement[] | NativeError;
public abstract testGrabElsAsJson(decode?: boolean): IGrabbedElement[] | NativeError;

// Used only for testing and debug
public abstract testGrabElsAsProto(): IGrabbedElement[] | NativeError;
public abstract testGrabElsAsProto(decode?: boolean): IGrabbedElement[] | NativeError;
}

export abstract class RustSessionNative {
Expand Down Expand Up @@ -879,8 +879,12 @@ export class RustSessionWrapper extends RustSession {
}

// Used only for testing and debug
public testGrabElsAsJson(): IGrabbedElement[] | NativeError {
public testGrabElsAsJson(decode?: boolean): IGrabbedElement[] | NativeError {
try {
const received = this._native.testGrabElsAsJson();
if (decode === false) {
return [];
}
const lines: Array<{
c: string;
id: number;
Expand Down Expand Up @@ -911,9 +915,13 @@ export class RustSessionWrapper extends RustSession {
}

// Used only for testing and debug
public testGrabElsAsProto(): IGrabbedElement[] | NativeError {
public testGrabElsAsProto(decode?: boolean): IGrabbedElement[] | NativeError {
try {
return Types.decodeGrabbedElementList(this._native.testGrabElsAsProto());
const received = this._native.testGrabElsAsProto();
if (decode === false) {
return [];
}
return Types.decodeGrabbedElementList(received);
} catch (err) {
return new NativeError(new Error(utils.error(err)), Type.Other, Source.Other);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Started } from "./Started";
import type { Stopped } from "./Stopped";
import type { TicksWithUuid } from "./TicksWithUuid";
import type { Started } from './Started';
import type { Stopped } from './Stopped';
import type { TicksWithUuid } from './TicksWithUuid';

export type Transition = { "Started": Started } | { "Ticks": TicksWithUuid } | { "Stopped": Stopped };
export type Transition = { Started: Started } | { Ticks: TicksWithUuid } | { Stopped: Stopped };
29 changes: 29 additions & 0 deletions application/apps/rustcore/ts-bindings/src/protocol/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,35 @@ export function toObserveOptions(source: IObserve): ObserveOptions {
return { origin: { origin }, parser };
}

export function decodeLifecycleTransition(buf: number[]): any {
const event: LifecycleTransition = proto.LifecycleTransition.decode(Uint8Array.from(buf));
if (!event.transition) {
return {};
}
const inner: Transition = event.transition;
if ('Started' in inner) {
return { Started: { uuid: inner.Started.uuid, alias: inner.Started.alias } };
} else if ('Ticks' in inner) {
const ticks = inner.Ticks.ticks;
return {
Ticks: {
uuid: inner.Ticks.uuid,
progress:
ticks === null
? {}
: {
count: Number(ticks.count),
state: ticks.state,
total: Number(ticks.total),
},
},
};
} else if ('Stopped' in inner) {
return { Stopped: inner.Stopped.uuid };
} else {
throw new Error(`Fail to parse event: ${JSON.stringify(event)}`);
}
}
export function decodeCallbackEvent(buf: number[]): any {
const event: CallbackEvent = proto.CallbackEvent.decode(Uint8Array.from(buf));
if (!event.event) {
Expand Down
11 changes: 7 additions & 4 deletions application/apps/rustcore/ts-bindings/src/provider/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,20 @@ import { Logger } from 'platform/log';
import { scope } from 'platform/env/scope';
import { TEventData, TEventEmitter, IEventData } from '../provider/provider.general';

import * as Types from '../protocol';

export interface IOrderStat {
type: 'E' | 'O';
name: string;
id: string | undefined;
emitted: number; // Time of emitting event or operation
duration: number;
}

export type Decoder = (buf: number[]) => any;

export abstract class Computation<TEvents, IEventsSignatures, IEventsInterfaces> {
private _destroyed: boolean = false;
private readonly _uuid: string;
private readonly _decoder: Decoder;
private readonly _tracking: {
subjects: {
unsupported: Subject<string>;
Expand Down Expand Up @@ -56,8 +58,9 @@ export abstract class Computation<TEvents, IEventsSignatures, IEventsInterfaces>
};
public readonly logger: Logger;

constructor(uuid: string) {
constructor(uuid: string, decoder: Decoder) {
this._uuid = uuid;
this._decoder = decoder;
this._emitter = this._emitter.bind(this);
this.logger = scope.getLogger(`${this.getName()}: ${uuid}`);
}
Expand Down Expand Up @@ -252,7 +255,7 @@ export abstract class Computation<TEvents, IEventsSignatures, IEventsInterfaces>
let event: Required<IEventData>;
try {
if (data instanceof Array) {
event = Types.decodeCallbackEvent(data);
event = this._decoder(data);
} else if (typeof data === 'string') {
try {
event = JSON.parse(data);
Expand Down