From 2db897deb54c27a55b55ca63b8f5c9c491178c7f Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Thu, 5 Oct 2023 14:52:44 -0400 Subject: [PATCH 1/7] Improve SourceMap and support columns --- src/logic/sourcemap.ts | 93 ++++++++++++++++++++++----------- src/main.ts | 2 +- tests/8.LogicSig.ts | 113 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 30 deletions(-) diff --git a/src/logic/sourcemap.ts b/src/logic/sourcemap.ts index b00564911..814de3e2c 100644 --- a/src/logic/sourcemap.ts +++ b/src/logic/sourcemap.ts @@ -1,13 +1,32 @@ import * as vlq from 'vlq'; +/** + * Represents a location in a source file. + */ +export interface SourceLocation { + line: number; + column: number; +} + +/** + * Represents the location of a specific PC in a source line. + */ +export interface PcLineLocation { + pc: number; + column: number; +} + +/** + * Contains a mapping from TEAL program PC to source file location. + */ export class SourceMap { - version: number; - sources: string[]; - names: string[]; - mappings: string; + public readonly version: number; + public readonly sources: string[]; + public readonly names: string[]; + public readonly mappings: string; - pcToLine: { [key: number]: number }; - lineToPc: { [key: number]: number[] }; + private pcToLocation: Map; + private lineToPc: Map; constructor({ version, @@ -33,35 +52,51 @@ export class SourceMap { 'mapping undefined, cannot build source map without `mapping`' ); - const pcList = this.mappings.split(';').map((m) => { - const decoded = vlq.decode(m); - if (decoded.length > 2) return decoded[2]; - return undefined; - }); - - this.pcToLine = {}; - this.lineToPc = {}; - - let lastLine = 0; - for (const [pc, lineDelta] of pcList.entries()) { - // If the delta is not undefined, the lastLine should be updated with - // lastLine + the delta - if (lineDelta !== undefined) { - lastLine += lineDelta; - } + const pcList = this.mappings.split(';').map(vlq.decode); + + this.pcToLocation = new Map(); + this.lineToPc = new Map(); - if (!(lastLine in this.lineToPc)) this.lineToPc[lastLine] = []; + const lastLocation: SourceLocation = { + line: 0, + column: 0, + }; + for (const [pc, data] of pcList.entries()) { + if (data.length < 4) continue; - this.lineToPc[lastLine].push(pc); - this.pcToLine[pc] = lastLine; + const [, , lineDelta, columnDelta] = data; + + lastLocation.line += lineDelta; + lastLocation.column += columnDelta; + + let pcsForLine = this.lineToPc.get(lastLocation.line); + if (pcsForLine === undefined) { + pcsForLine = []; + this.lineToPc.set(lastLocation.line, pcsForLine); + } + + pcsForLine.push({ + pc, + column: lastLocation.column, + }); + this.pcToLocation.set(pc, { + line: lastLocation.line, + column: lastLocation.column, + }); } } - getLineForPc(pc: number): number | undefined { - return this.pcToLine[pc]; + getPcs(): number[] { + return Array.from(this.pcToLocation.keys()); + } + + getLocationForPc(pc: number): SourceLocation | undefined { + return this.pcToLocation.get(pc); } - getPcsForLine(line: number): number[] | undefined { - return this.lineToPc[line]; + getPcsForLine(line: number): PcLineLocation[] { + const pcs = this.lineToPc.get(line); + if (pcs === undefined) return []; + return pcs; } } diff --git a/src/main.ts b/src/main.ts index be5b92d12..392608ab3 100644 --- a/src/main.ts +++ b/src/main.ts @@ -183,7 +183,7 @@ export { verifyMultisig, multisigAddress, } from './multisig'; -export { SourceMap } from './logic/sourcemap'; +export { SourceMap, SourceLocation, PcLineLocation } from './logic/sourcemap'; export * from './dryrun'; export * from './makeTxn'; diff --git a/tests/8.LogicSig.ts b/tests/8.LogicSig.ts index c5db8fbf4..d1d172668 100644 --- a/tests/8.LogicSig.ts +++ b/tests/8.LogicSig.ts @@ -690,3 +690,116 @@ describe('signLogicSigTransaction', () => { assert.deepStrictEqual(actual, expected); }); }); + +describe('SourceMap', () => { + const input = { + version: 3, + sources: ['test/scripts/e2e_subs/tealprogs/sourcemap-test.teal'], + names: [], + mappings: + ';;;;AAGA;;AAAmB;;AAAO;AAAI;;;AAE9B;;AAAO;;AAAO;AACd;;AAAO;AAAO;AALO;AAAI;AASrB;AACA', + }; + + const expectedLocations = new Map([ + [4, { line: 3, column: 0 }], + [6, { line: 3, column: 19 }], + [8, { line: 3, column: 26 }], + [9, { line: 3, column: 30 }], + [12, { line: 5, column: 0 }], + [14, { line: 5, column: 7 }], + [16, { line: 5, column: 14 }], + [17, { line: 6, column: 0 }], + [19, { line: 6, column: 7 }], + [20, { line: 6, column: 14 }], + [21, { line: 1, column: 21 }], + [22, { line: 1, column: 25 }], + [23, { line: 10, column: 4 }], + [24, { line: 11, column: 4 }], + ]); + + const expectedPcsForLine = new Map([ + [ + 3, + [ + { pc: 4, column: 0 }, + { pc: 6, column: 19 }, + { pc: 8, column: 26 }, + { pc: 9, column: 30 }, + ], + ], + [ + 5, + [ + { pc: 12, column: 0 }, + { pc: 14, column: 7 }, + { pc: 16, column: 14 }, + ], + ], + [ + 6, + [ + { pc: 17, column: 0 }, + { pc: 19, column: 7 }, + { pc: 20, column: 14 }, + ], + ], + [ + 1, + [ + { pc: 21, column: 21 }, + { pc: 22, column: 25 }, + ], + ], + [10, [{ pc: 23, column: 4 }]], + [11, [{ pc: 24, column: 4 }]], + ]); + + it('should be able to read a SourceMap', () => { + const sourceMap = new algosdk.SourceMap(input); + + assert.strictEqual(sourceMap.version, input.version); + assert.deepStrictEqual(sourceMap.sources, input.sources); + assert.deepStrictEqual(sourceMap.names, input.names); + assert.strictEqual(sourceMap.mappings, input.mappings); + }); + + describe('getLocationForPc', () => { + it('should return the correct location for all pcs', () => { + const sourceMap = new algosdk.SourceMap(input); + const maxPcToCheck = 30; + + for (let pc = 0; pc < maxPcToCheck; pc++) { + const expected = expectedLocations.get(pc); + assert.deepStrictEqual( + sourceMap.getLocationForPc(pc), + expected, + `pc=${pc}` + ); + } + }); + }); + + describe('getPcs', () => { + it('should return the correct pcs', () => { + const sourceMap = new algosdk.SourceMap(input); + const expectedPcs = Array.from(expectedLocations.keys()); + assert.deepStrictEqual(sourceMap.getPcs(), expectedPcs); + }); + }); + + describe('getPcsForLine', () => { + it('should return the correct pcs for all lines', () => { + const sourceMap = new algosdk.SourceMap(input); + const maxLineToCheck = 15; + + for (let line = 1; line <= maxLineToCheck; line++) { + const expected = expectedPcsForLine.get(line) || []; + assert.deepStrictEqual( + sourceMap.getPcsForLine(line), + expected, + `line=${line}` + ); + } + }); + }); +}); From 327f33ca75dc7c931820844a674fe763f9b88a16 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Thu, 12 Oct 2023 16:36:43 -0400 Subject: [PATCH 2/7] Report source file index and name index --- src/logic/sourcemap.ts | 56 ++++++++++++++++++++++++++++++++---------- tests/8.LogicSig.ts | 30 +++++++++++----------- 2 files changed, 58 insertions(+), 28 deletions(-) diff --git a/src/logic/sourcemap.ts b/src/logic/sourcemap.ts index 814de3e2c..8d2205153 100644 --- a/src/logic/sourcemap.ts +++ b/src/logic/sourcemap.ts @@ -6,6 +6,8 @@ import * as vlq from 'vlq'; export interface SourceLocation { line: number; column: number; + sourceIndex: number; + nameIndex?: number; } /** @@ -14,6 +16,7 @@ export interface SourceLocation { export interface PcLineLocation { pc: number; column: number; + nameIndex?: number; } /** @@ -21,12 +24,23 @@ export interface PcLineLocation { */ export class SourceMap { public readonly version: number; + /** + * A list of original sources used by the "mappings" entry. + */ public readonly sources: string[]; + /** + * A list of symbol names used by the "mappings" entry. + */ public readonly names: string[]; + /** + * A string with the encoded mapping data. + */ public readonly mappings: string; private pcToLocation: Map; - private lineToPc: Map; + + // Key is `${sourceIndex}:${line}` + private sourceAndLineToPc: Map; constructor({ version, @@ -55,34 +69,50 @@ export class SourceMap { const pcList = this.mappings.split(';').map(vlq.decode); this.pcToLocation = new Map(); - this.lineToPc = new Map(); + this.sourceAndLineToPc = new Map(); const lastLocation: SourceLocation = { line: 0, column: 0, + sourceIndex: 0, + nameIndex: 0, }; for (const [pc, data] of pcList.entries()) { if (data.length < 4) continue; - const [, , lineDelta, columnDelta] = data; + const nameDelta = data.length > 4 ? data[4] : undefined; + const [, sourceDelta, lineDelta, columnDelta] = data; + lastLocation.sourceIndex += sourceDelta; lastLocation.line += lineDelta; lastLocation.column += columnDelta; + if (typeof nameDelta !== 'undefined') { + lastLocation.nameIndex += nameDelta; + } - let pcsForLine = this.lineToPc.get(lastLocation.line); - if (pcsForLine === undefined) { - pcsForLine = []; - this.lineToPc.set(lastLocation.line, pcsForLine); + const sourceAndLineKey = `${lastLocation.sourceIndex}:${lastLocation.line}`; + let pcsForSourceAndLine = this.sourceAndLineToPc.get(sourceAndLineKey); + if (pcsForSourceAndLine === undefined) { + pcsForSourceAndLine = []; + this.sourceAndLineToPc.set(sourceAndLineKey, pcsForSourceAndLine); } - pcsForLine.push({ + const pcInLine: PcLineLocation = { pc, column: lastLocation.column, - }); - this.pcToLocation.set(pc, { + }; + const pcLocation: SourceLocation = { line: lastLocation.line, column: lastLocation.column, - }); + sourceIndex: lastLocation.sourceIndex, + }; + if (typeof nameDelta !== 'undefined') { + pcInLine.nameIndex = lastLocation.nameIndex; + pcLocation.nameIndex = lastLocation.nameIndex; + } + + pcsForSourceAndLine.push(pcInLine); + this.pcToLocation.set(pc, pcLocation); } } @@ -94,8 +124,8 @@ export class SourceMap { return this.pcToLocation.get(pc); } - getPcsForLine(line: number): PcLineLocation[] { - const pcs = this.lineToPc.get(line); + getPcsOnSourceLine(sourceIndex: number, line: number): PcLineLocation[] { + const pcs = this.sourceAndLineToPc.get(`${sourceIndex}:${line}`); if (pcs === undefined) return []; return pcs; } diff --git a/tests/8.LogicSig.ts b/tests/8.LogicSig.ts index d1d172668..179c8fd90 100644 --- a/tests/8.LogicSig.ts +++ b/tests/8.LogicSig.ts @@ -701,20 +701,20 @@ describe('SourceMap', () => { }; const expectedLocations = new Map([ - [4, { line: 3, column: 0 }], - [6, { line: 3, column: 19 }], - [8, { line: 3, column: 26 }], - [9, { line: 3, column: 30 }], - [12, { line: 5, column: 0 }], - [14, { line: 5, column: 7 }], - [16, { line: 5, column: 14 }], - [17, { line: 6, column: 0 }], - [19, { line: 6, column: 7 }], - [20, { line: 6, column: 14 }], - [21, { line: 1, column: 21 }], - [22, { line: 1, column: 25 }], - [23, { line: 10, column: 4 }], - [24, { line: 11, column: 4 }], + [4, { sourceIndex: 0, line: 3, column: 0 }], + [6, { sourceIndex: 0, line: 3, column: 19 }], + [8, { sourceIndex: 0, line: 3, column: 26 }], + [9, { sourceIndex: 0, line: 3, column: 30 }], + [12, { sourceIndex: 0, line: 5, column: 0 }], + [14, { sourceIndex: 0, line: 5, column: 7 }], + [16, { sourceIndex: 0, line: 5, column: 14 }], + [17, { sourceIndex: 0, line: 6, column: 0 }], + [19, { sourceIndex: 0, line: 6, column: 7 }], + [20, { sourceIndex: 0, line: 6, column: 14 }], + [21, { sourceIndex: 0, line: 1, column: 21 }], + [22, { sourceIndex: 0, line: 1, column: 25 }], + [23, { sourceIndex: 0, line: 10, column: 4 }], + [24, { sourceIndex: 0, line: 11, column: 4 }], ]); const expectedPcsForLine = new Map([ @@ -795,7 +795,7 @@ describe('SourceMap', () => { for (let line = 1; line <= maxLineToCheck; line++) { const expected = expectedPcsForLine.get(line) || []; assert.deepStrictEqual( - sourceMap.getPcsForLine(line), + sourceMap.getPcsOnSourceLine(0, line), expected, `line=${line}` ); From a1e40e4db34d9c63b3bfef4dd21324eef9d15be8 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 15 Nov 2023 13:31:35 -0500 Subject: [PATCH 3/7] Update cucumber tests --- .test-env | 2 +- tests/cucumber/steps/steps.js | 40 ++++++++++++++++++++--------------- tests/cucumber/unit.tags | 2 +- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/.test-env b/.test-env index a321a7f1e..1ad8e1846 100644 --- a/.test-env +++ b/.test-env @@ -1,6 +1,6 @@ # Configs for testing repo download: SDK_TESTING_URL="https://github.com/algorand/algorand-sdk-testing" -SDK_TESTING_BRANCH="V2" +SDK_TESTING_BRANCH="sourcemap-v2-testcases" SDK_TESTING_HARNESS="test-harness" INSTALL_ONLY=0 diff --git a/tests/cucumber/steps/steps.js b/tests/cucumber/steps/steps.js index 02b52ff5d..43085d62c 100644 --- a/tests/cucumber/steps/steps.js +++ b/tests/cucumber/steps/steps.js @@ -4697,29 +4697,35 @@ module.exports = function getSteps(options) { this.sourcemap = new algosdk.SourceMap(js); }); - Then( - 'the string composed of pc:line number equals {string}', - function (mapping) { - const buff = Object.entries(this.sourcemap.pcToLine).map( - ([pc, line]) => `${pc}:${line}` - ); - assert.deepStrictEqual(buff.join(';'), mapping); - } - ); + Then('the source map contains pcs {string}', function (pcsString) { + const pcs = pcsString.split(',').map((pc) => parseInt(pc, 10)); + assert.deepStrictEqual(this.sourcemap.getPcs(), pcs); + }); Then( - 'getting the line associated with a pc {string} equals {string}', - function (pc, expectedLine) { - const actualLine = this.sourcemap.getLineForPc(parseInt(pc)); - assert.deepStrictEqual(actualLine, parseInt(expectedLine)); + 'the source map maps pc {int} to line {int} and column {int} of source {string}', + function (pc, expectedLine, expectedColumn, source) { + const actual = this.sourcemap.getLocationForPc(pc); + assert.ok(actual); + assert.strictEqual(actual.line, expectedLine); + assert.strictEqual(actual.column, expectedColumn); + assert.strictEqual(this.sourcemap.sources[actual.sourceIndex], source); } ); Then( - 'getting the last pc associated with a line {string} equals {string}', - function (line, expectedPc) { - const actualPcs = this.sourcemap.getPcsForLine(parseInt(line)); - assert.deepStrictEqual(actualPcs.pop(), parseInt(expectedPc)); + 'the source map maps source {string} and line {int} to pc {int} at column {int}', + function (source, line, pc, expectedColumn) { + const sourceIndex = this.sourcemap.sources.indexOf(source); + assert.ok(sourceIndex >= 0); + const actualPcs = this.sourcemap.getPcsOnSourceLine(sourceIndex, line); + for (const actualPcInfo of actualPcs) { + if (actualPcInfo.pc === pc) { + assert.strictEqual(actualPcInfo.column, expectedColumn); + return; + } + } + throw new Error(`Could not find pc ${pc}`); } ); diff --git a/tests/cucumber/unit.tags b/tests/cucumber/unit.tags index 5c66cad6c..1dd481160 100644 --- a/tests/cucumber/unit.tags +++ b/tests/cucumber/unit.tags @@ -27,7 +27,7 @@ @unit.responses.timestamp @unit.responses.txid.json @unit.responses.txngroupdeltas.json -@unit.sourcemap +@unit.sourcemapv2 @unit.stateproof.paths @unit.stateproof.responses @unit.stateproof.responses.msgp From 7d4b4b0911ff9410a9e822757c29e98bde48a6e5 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 15 Nov 2023 13:49:31 -0500 Subject: [PATCH 4/7] Rename to ProgramSourceMap --- src/logic/sourcemap.ts | 2 +- src/main.ts | 6 +++++- tests/cucumber/steps/steps.js | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/logic/sourcemap.ts b/src/logic/sourcemap.ts index 8d2205153..932681c06 100644 --- a/src/logic/sourcemap.ts +++ b/src/logic/sourcemap.ts @@ -22,7 +22,7 @@ export interface PcLineLocation { /** * Contains a mapping from TEAL program PC to source file location. */ -export class SourceMap { +export class ProgramSourceMap { public readonly version: number; /** * A list of original sources used by the "mappings" entry. diff --git a/src/main.ts b/src/main.ts index 392608ab3..26ed02cfe 100644 --- a/src/main.ts +++ b/src/main.ts @@ -183,7 +183,11 @@ export { verifyMultisig, multisigAddress, } from './multisig'; -export { SourceMap, SourceLocation, PcLineLocation } from './logic/sourcemap'; +export { + ProgramSourceMap, + SourceLocation, + PcLineLocation, +} from './logic/sourcemap'; export * from './dryrun'; export * from './makeTxn'; diff --git a/tests/cucumber/steps/steps.js b/tests/cucumber/steps/steps.js index 43085d62c..6a4781b51 100644 --- a/tests/cucumber/steps/steps.js +++ b/tests/cucumber/steps/steps.js @@ -4694,7 +4694,7 @@ module.exports = function getSteps(options) { Given('a source map json file {string}', async function (srcmap) { const js = await loadResourceAsJson(srcmap); - this.sourcemap = new algosdk.SourceMap(js); + this.sourcemap = new algosdk.ProgramSourceMap(js); }); Then('the source map contains pcs {string}', function (pcsString) { From 7451256b5fe87901101479a033174caab082e8e6 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 15 Nov 2023 13:50:53 -0500 Subject: [PATCH 5/7] Firefox fix --- tests/cucumber/steps/steps.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/cucumber/steps/steps.js b/tests/cucumber/steps/steps.js index 6a4781b51..95dedc706 100644 --- a/tests/cucumber/steps/steps.js +++ b/tests/cucumber/steps/steps.js @@ -4698,8 +4698,11 @@ module.exports = function getSteps(options) { }); Then('the source map contains pcs {string}', function (pcsString) { - const pcs = pcsString.split(',').map((pc) => parseInt(pc, 10)); - assert.deepStrictEqual(this.sourcemap.getPcs(), pcs); + const expectedPcs = makeArray( + ...pcsString.split(',').map((pc) => parseInt(pc, 10)) + ); + const actualPcs = makeArray(...this.sourcemap.getPcs()); + assert.deepStrictEqual(actualPcs, expectedPcs); }); Then( From a0700c65474b1b6681ec035f62e72ceb22b2ba85 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 15 Nov 2023 14:03:25 -0500 Subject: [PATCH 6/7] Update unit tests --- tests/8.LogicSig.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/8.LogicSig.ts b/tests/8.LogicSig.ts index 179c8fd90..7cff3ed59 100644 --- a/tests/8.LogicSig.ts +++ b/tests/8.LogicSig.ts @@ -691,7 +691,7 @@ describe('signLogicSigTransaction', () => { }); }); -describe('SourceMap', () => { +describe('ProgramSourceMap', () => { const input = { version: 3, sources: ['test/scripts/e2e_subs/tealprogs/sourcemap-test.teal'], @@ -754,8 +754,8 @@ describe('SourceMap', () => { [11, [{ pc: 24, column: 4 }]], ]); - it('should be able to read a SourceMap', () => { - const sourceMap = new algosdk.SourceMap(input); + it('should be able to read a ProgramSourceMap', () => { + const sourceMap = new algosdk.ProgramSourceMap(input); assert.strictEqual(sourceMap.version, input.version); assert.deepStrictEqual(sourceMap.sources, input.sources); @@ -765,7 +765,7 @@ describe('SourceMap', () => { describe('getLocationForPc', () => { it('should return the correct location for all pcs', () => { - const sourceMap = new algosdk.SourceMap(input); + const sourceMap = new algosdk.ProgramSourceMap(input); const maxPcToCheck = 30; for (let pc = 0; pc < maxPcToCheck; pc++) { @@ -781,7 +781,7 @@ describe('SourceMap', () => { describe('getPcs', () => { it('should return the correct pcs', () => { - const sourceMap = new algosdk.SourceMap(input); + const sourceMap = new algosdk.ProgramSourceMap(input); const expectedPcs = Array.from(expectedLocations.keys()); assert.deepStrictEqual(sourceMap.getPcs(), expectedPcs); }); @@ -789,7 +789,7 @@ describe('SourceMap', () => { describe('getPcsForLine', () => { it('should return the correct pcs for all lines', () => { - const sourceMap = new algosdk.SourceMap(input); + const sourceMap = new algosdk.ProgramSourceMap(input); const maxLineToCheck = 15; for (let line = 1; line <= maxLineToCheck; line++) { From ccf6c682eb32b5109b52d7e6150ebae17c33715d Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Mon, 20 Nov 2023 12:42:44 -0500 Subject: [PATCH 7/7] Update .test-env --- .test-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-env b/.test-env index 1ad8e1846..a321a7f1e 100644 --- a/.test-env +++ b/.test-env @@ -1,6 +1,6 @@ # Configs for testing repo download: SDK_TESTING_URL="https://github.com/algorand/algorand-sdk-testing" -SDK_TESTING_BRANCH="sourcemap-v2-testcases" +SDK_TESTING_BRANCH="V2" SDK_TESTING_HARNESS="test-harness" INSTALL_ONLY=0