From 5ff5ad4423d3f4464e2454d6491c05f369e4b963 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 28 Oct 2024 22:54:59 +0800 Subject: [PATCH 1/9] remove ic --- compiler/ccgtypes.nim | 1 - compiler/cgen.nim | 35 ++------------------ compiler/importer.nim | 1 - compiler/liftdestructors.nim | 2 -- compiler/lookups.nim | 4 --- compiler/magicsys.nim | 2 -- compiler/main.nim | 26 +++------------ compiler/modulegraphs.nim | 47 +-------------------------- compiler/passes.nim | 3 -- compiler/pipelines.nim | 6 ---- compiler/pragmas.nim | 4 --- compiler/sem.nim | 3 -- compiler/semdata.nim | 62 +----------------------------------- compiler/semexprs.nim | 1 - compiler/seminst.nim | 8 +---- compiler/semstmts.nim | 1 - testament/categories.nim | 2 +- 17 files changed, 10 insertions(+), 198 deletions(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 1c1936db6aed..cd1818dcb57d 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -2021,7 +2021,6 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope = owner = m.module.position.int32 m.g.typeInfoMarker[sig] = (str: result, owner: owner) - rememberEmittedTypeInfo(m.g.graph, FileIndex(owner), $result) case t.kind of tyEmpty, tyVoid: result = cIntValue(0) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index c4931d7ba74d..0a2ec10a139d 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -30,7 +30,6 @@ when not defined(leanCompiler): import std/strutils except `%`, addf # collides with ropes.`%` -from ic / ic import ModuleBackendFlag import std/[dynlib, math, tables, sets, os, intsets, hashes] const @@ -1366,8 +1365,7 @@ proc genProcNoForward(m: BModule, prc: PSym) = #if prc.loc.k == locNone: # mangle the inline proc based on the module where it is defined - # not on the first module that uses it - let m2 = if m.config.symbolFiles != disabledSf: m - else: findPendingModule(m, prc) + let m2 = findPendingModule(m, prc) fillProcLoc(m2, prc.ast[namePos]) #elif {sfExportc, sfImportc} * prc.flags == {}: # # reset name to restore consistency in case of hashing collisions: @@ -1719,33 +1717,6 @@ proc genMainProc(m: BModule) = if m.config.cppCustomNamespace.len > 0: openNamespaceNim(m.config.cppCustomNamespace, m.s[cfsProcs]) -proc registerInitProcs*(g: BModuleList; m: PSym; flags: set[ModuleBackendFlag]) = - ## Called from the IC backend. - if HasDatInitProc in flags: - let datInit = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "DatInit000" - g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit]) - g.mainDatInit.addf("\t$1();$N", [datInit]) - if HasModuleInitProc in flags: - let init = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "Init000" - g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init]) - let initCall = "\t$1();$N" % [init] - if sfMainModule in m.flags: - g.mainModInit.add(initCall) - elif sfSystemModule in m.flags: - g.mainDatInit.add(initCall) # systemInit must called right after systemDatInit if any - else: - g.otherModsInit.add(initCall) - -proc whichInitProcs*(m: BModule): set[ModuleBackendFlag] = - # called from IC. - result = {} - if m.hcrOn or m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0: - result.incl HasModuleInitProc - for i in cfsTypeInit1..cfsDynLibInit: - if m.s[i].len != 0: - result.incl HasDatInitProc - break - proc registerModuleToMain(g: BModuleList; m: BModule) = let init = m.getInitName @@ -1840,7 +1811,6 @@ proc genDatInitCode(m: BModule) = if moduleDatInitRequired: m.s[cfsDatInitProc].add(prc) - #rememberFlag(m.g.graph, m.module, HasDatInitProc) # Very similar to the contents of symInDynamicLib - basically only the # things needed for the hot code reloading runtime procs to be loaded @@ -1971,7 +1941,6 @@ proc genInitCode(m: BModule) = if moduleInitRequired or sfMainModule in m.module.flags: m.s[cfsInitProc].add(prc) - #rememberFlag(m.g.graph, m.module, HasModuleInitProc) genDatInitCode(m) @@ -2234,7 +2203,7 @@ proc writeModule(m: BModule, pending: bool) = var cf = Cfile(nimname: m.module.name.s, cname: cfile, obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {}) var code = genModule(m, cf) - if code != "" or m.config.symbolFiles != disabledSf: + if code != "": when hasTinyCBackend: if m.config.cmd == cmdTcc: tccgen.compileCCode($code, m.config) diff --git a/compiler/importer.nim b/compiler/importer.nim index ffb7e0305651..a02cd2931148 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -275,7 +275,6 @@ proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym = n = transf.node let f = checkModuleName(c.config, n) if f != InvalidFileIdx: - addImportFileDep(c, f) let L = c.graph.importStack.len let recursion = c.graph.importStack.find(f) c.graph.importStack.add f diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 69d457a566e5..f509ba1b05e6 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -1207,8 +1207,6 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; result.ast[pragmasPos].add newTree(nkExprColonExpr, newIdentNode(g.cache.getIdent("raises"), info), newNodeI(nkBracket, info)) - completePartialOp(g, idgen.module, typ, kind, result) - proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym, info: TLineInfo; idgen: IdGenerator): PSym = diff --git a/compiler/lookups.nim b/compiler/lookups.nim index d8fcf73e0f73..b6ecbe235f8b 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -410,8 +410,6 @@ proc addDecl*(c: PContext, sym: PSym) {.inline.} = proc addPrelimDecl*(c: PContext, sym: PSym) = discard c.currentScope.addUniqueSym(sym) -from ic / ic import addHidden - proc addInterfaceDeclAux(c: PContext, sym: PSym) = ## adds symbol to the module for either private or public access. if sfExported in sym.flags: @@ -420,8 +418,6 @@ proc addInterfaceDeclAux(c: PContext, sym: PSym) = else: internalError(c.config, sym.info, "addInterfaceDeclAux") elif sym.kind in ExportableSymKinds and c.module != nil and isTopLevelInsideDeclaration(c, sym): strTableAdd(semtabAll(c.graph, c.module), sym) - if c.config.symbolFiles != disabledSf: - addHidden(c.encoder, c.packedRepr, sym) proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) = ## adds a symbol on the scope and the interface if appropriate diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index 47a71d56cd63..2de3c496f1d0 100644 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -113,8 +113,6 @@ proc makeVarType*(owner: PSym; baseType: PType; idgen: IdGenerator; kind = tyVar proc getCompilerProc*(g: ModuleGraph; name: string): PSym = let ident = getIdent(g.cache, name) result = strTableGet(g.compilerprocs, ident) - if result == nil: - result = loadCompilerProc(g, name) proc registerCompilerProc*(g: ModuleGraph; s: PSym) = strTableAdd(g.compilerprocs, s) diff --git a/compiler/main.nim b/compiler/main.nim index 4c52317cfae7..f3362e5c516c 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -26,7 +26,6 @@ import when defined(nimPreviewSlimSystem): import std/[syncio, assertions] -import ic / [cbackend, integrity, navigator, ic] import ../dist/checksums/src/checksums/sha1 @@ -95,14 +94,6 @@ proc commandCheck(graph: ModuleGraph) = setPipeLinePass(graph, SemPass) compilePipelineProject(graph) - if conf.symbolFiles != disabledSf: - case conf.ideCmd - of ideDef: navDefinition(graph) - of ideUse: navUsages(graph) - of ideDus: navDefusages(graph) - else: discard - writeRodFiles(graph) - when not defined(leanCompiler): proc commandDoc2(graph: ModuleGraph; ext: string) = handleDocOutputOptions graph.config @@ -137,15 +128,7 @@ proc commandCompileToC(graph: ModuleGraph) = compilePipelineProject(graph) if graph.config.errorCounter > 0: return # issue #9933 - if conf.symbolFiles == disabledSf: - cgenWriteModules(graph.backend, conf) - else: - if isDefined(conf, "nimIcIntegrityChecks"): - checkIntegrity(graph) - generateCode(graph) - # graph.backend can be nil under IC when nothing changed at all: - if graph.backend != nil: - cgenWriteModules(graph.backend, conf) + cgenWriteModules(graph.backend, conf) if conf.cmd != cmdTcc and graph.backend != nil: extccomp.callCCompiler(conf) # for now we do not support writing out a .json file with the build instructions when HCR is on @@ -206,8 +189,9 @@ proc commandScan(cache: IdentCache, config: ConfigRef) = rawMessage(config, errGenerated, "cannot open file: " & f.string) proc commandView(graph: ModuleGraph) = - let f = toAbsolute(mainCommandArg(graph.config), AbsoluteDir getCurrentDir()).addFileExt(RodExt) - rodViewer(f, graph.config, graph.cache) + # let f = toAbsolute(mainCommandArg(graph.config), AbsoluteDir getCurrentDir()).addFileExt(RodExt) + # rodViewer(f, graph.config, graph.cache) + discard const PrintRopeCacheStats = false @@ -305,8 +289,6 @@ proc mainCommand*(graph: ModuleGraph) = case conf.cmd of cmdBackends: compileToBackend() - when BenchIC: - echoTimes graph.packed of cmdTcc: when hasTinyCBackend: extccomp.setCC(conf, "tcc", unknownLineInfo) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index dd6a590e4fc7..4ad73132a61b 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -224,13 +224,6 @@ proc simulateCachedModule(g: ModuleGraph; moduleSym: PSym; m: PackedModule) = echo "simulating ", moduleSym.name.s, " ", moduleSym.position simulateLoadedModule(g.packed, g.config, g.cache, moduleSym, m) -proc initEncoder*(g: ModuleGraph; module: PSym) = - let id = module.position - if id >= g.encoders.len: - setLen g.encoders, id+1 - ic.initEncoder(g.encoders[id], - g.packed[id].fromDisk, module, g.config, g.startupPackedConfig) - type ModuleIter* = object fromRod: bool @@ -365,13 +358,6 @@ proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttac ## we also need to record this to the packed module. g.attachedOps[op][t.itemId] = LazySym(sym: value) -proc completePartialOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) = - if g.config.symbolFiles != disabledSf: - assert module < g.encoders.len - assert isActive(g.encoders[module]) - toPackedGeneratedProcDef(value, g.encoders[module], g.packed[module].fromDisk) - #storeAttachedProcDef(t, op, value, g.encoders[module], g.packed[module].fromDisk) - iterator getDispatchers*(g: ModuleGraph): PSym = for i in g.dispatchers.mitems: yield resolveSym(g, i) @@ -418,20 +404,6 @@ proc copyTypeProps*(g: ModuleGraph; module: int; dest, src: PType) = if op != nil: setAttachedOp(g, module, dest, k, op) -proc loadCompilerProc*(g: ModuleGraph; name: string): PSym = - result = nil - if g.config.symbolFiles == disabledSf: return nil - - # slow, linear search, but the results are cached: - for module in 0..= 0: - result = loadSymFromId(g.config, g.cache, g.packed, module, toPackedItemId(x)) - if result != nil: - strTableAdd(g.compilerprocs, result) - return result - proc loadPackedSym*(g: ModuleGraph; s: var LazySym) = if s.sym == nil: s.sym = loadSymFromId(g.config, g.cache, g.packed, s.id.module, s.id.packed) @@ -606,24 +578,7 @@ proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym = result = g.ifaces[fileIdx.int32].module proc moduleOpenForCodegen*(g: ModuleGraph; m: FileIndex): bool {.inline.} = - if g.config.symbolFiles == disabledSf: - result = true - else: - result = g.packed[m.int32].status notin {undefined, stored, loaded} - -proc rememberEmittedTypeInfo*(g: ModuleGraph; m: FileIndex; ti: string) = - #assert(not isCachedModule(g, m.int32)) - if g.config.symbolFiles != disabledSf: - #assert g.encoders[m.int32].isActive - assert g.packed[m.int32].status != stored - g.packed[m.int32].fromDisk.emittedTypeInfo.add ti - #echo "added typeinfo ", m.int32, " ", ti, " suspicious ", not g.encoders[m.int32].isActive - -proc rememberFlag*(g: ModuleGraph; m: PSym; flag: ModuleBackendFlag) = - if g.config.symbolFiles != disabledSf: - #assert g.encoders[m.int32].isActive - assert g.packed[m.position].status != stored - g.packed[m.position].fromDisk.backendFlags.incl flag + result = true proc closeRodFile*(g: ModuleGraph; m: PSym) = if g.config.symbolFiles in {readOnlySf, v2Sf}: diff --git a/compiler/passes.nim b/compiler/passes.nim index d6b141078613..761daeeb0119 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -19,7 +19,6 @@ import modules, pathutils, packages, sem, semdata -import ic/replayer export skipCodegen, resolveMod, prepareConfigNotes @@ -185,8 +184,6 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fr partialInitModule(result, graph, fileIdx, filename) for m in cachedModules: registerModuleById(graph, m) - replayStateChanges(graph.packed.pm[m.int].module, graph) - replayGenericCacheInformation(graph, m.int) elif graph.isDirty(result): result.flags.excl sfDirty # reset module fields: diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 5fddb046f074..52b3a7f6da49 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -12,7 +12,6 @@ when not defined(leanCompiler): import std/[syncio, objectdollar, assertions, tables, strutils, strtabs] import renderer -import ic/replayer proc setPipeLinePass*(graph: ModuleGraph; pass: PipelinePass) = graph.pipelinePass = pass @@ -251,11 +250,6 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF partialInitModule(result, graph, fileIdx, filename) for m in cachedModules: registerModuleById(graph, m) - if sfMainModule in flags and graph.config.cmd == cmdM: - discard - else: - replayStateChanges(graph.packed.pm[m.int].module, graph) - replayGenericCacheInformation(graph, m.int) elif graph.isDirty(result): result.flags.excl sfDirty # reset module fields: diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index c38fa2e62af6..7b97d4f1360d 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -21,7 +21,6 @@ import std/[os, math, strutils] when defined(nimPreviewSlimSystem): import std/assertions -from ic / ic import addCompilerProc const FirstCallConv* = wNimcall @@ -113,7 +112,6 @@ proc recordPragma(c: PContext; n: PNode; args: varargs[string]) = var recorded = newNodeI(nkReplayAction, n.info) for i in 0..args.high: recorded.add newStrNode(args[i], n.info) - addPragmaComputation(c, recorded) const errStringLiteralExpected = "string literal expected" @@ -764,8 +762,6 @@ proc markCompilerProc(c: PContext; s: PSym) = incl(s.flags, sfCompilerProc) incl(s.flags, sfUsed) registerCompilerProc(c.graph, s) - if c.config.symbolFiles != disabledSf: - addCompilerProc(c.encoder, c.packedRepr, s) proc deprecatedStmt(c: PContext; outerPragma: PNode) = let pragma = outerPragma[1] diff --git a/compiler/sem.nim b/compiler/sem.nim index 2ff4d758d2eb..a116a28db11b 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -505,7 +505,6 @@ const proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym, flags: TExprFlags = {}; expectedType: PType = nil): PNode = - rememberExpansion(c, nOrig.info, sym) pushInfoContext(c.config, nOrig.info, sym.detailedInfo) let info = getCallLineInfo(n) @@ -848,7 +847,6 @@ proc semWithPContext*(c: PContext, n: PNode): PNode = else: result = newNodeI(nkEmpty, n.info) #if c.config.cmd == cmdIdeTools: findSuggest(c, n) - storeRodNode(c, result) proc reportUnusedModules(c: PContext) = @@ -871,4 +869,3 @@ proc closePContext*(graph: ModuleGraph; c: PContext, n: PNode): PNode = result.add(c.module.ast) popOwner(c) popProcCon(c) - sealRodFile(c) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 688cd97009c8..87adaac39126 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -18,8 +18,6 @@ import options, ast, msgs, idents, renderer, magicsys, vmdef, modulegraphs, lineinfos, pathutils, layeredtable -import ic / ic - type TOptionEntry* = object # entries to put on a stack for pragma parsing options*: TOptions @@ -327,28 +325,6 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext = result.graph = graph result.signatures = initStrTable() result.features = graph.config.features - if graph.config.symbolFiles != disabledSf: - let id = module.position - if graph.config.cmd != cmdM: - assert graph.packed[id].status in {undefined, outdated} - graph.packed[id].status = storing - graph.packed[id].module = module - initEncoder graph, module - -template packedRepr*(c): untyped = c.graph.packed[c.module.position].fromDisk -template encoder*(c): untyped = c.graph.encoders[c.module.position] - -proc addIncludeFileDep*(c: PContext; f: FileIndex) = - if c.config.symbolFiles != disabledSf: - addIncludeFileDep(c.encoder, c.packedRepr, f) - -proc addImportFileDep*(c: PContext; f: FileIndex) = - if c.config.symbolFiles != disabledSf: - addImportFileDep(c.encoder, c.packedRepr, f) - -proc addPragmaComputation*(c: PContext; n: PNode) = - if c.config.symbolFiles != disabledSf: - addPragmaComputation(c.encoder, c.packedRepr, n) proc inclSym(sq: var seq[PSym], s: PSym): bool = for i in 0.. Date: Wed, 30 Oct 2024 22:16:56 +0800 Subject: [PATCH 2/9] progress --- compiler/importer.nim | 1 - compiler/modulegraphs.nim | 122 +++++--------------------------------- compiler/passes.nim | 23 ++----- compiler/pipelines.nim | 27 ++------- 4 files changed, 23 insertions(+), 150 deletions(-) diff --git a/compiler/importer.nim b/compiler/importer.nim index a02cd2931148..5b77ca8601a4 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -322,7 +322,6 @@ proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym = result = nil proc afterImport(c: PContext, m: PSym) = - if isCachedModule(c.graph, m): return # fixes bug #17510, for re-exported symbols let realModuleId = c.importModuleMap[m.id] for s in allSyms(c.graph, m): diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 4ad73132a61b..129c620c83ca 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -70,7 +70,6 @@ type ModuleGraph* {.acyclic.} = ref object ifaces*: seq[Iface] ## indexed by int32 fileIdx packed*: PackedModuleGraph - encoders*: seq[PackedEncoder] typeInstCache*: Table[ItemId, seq[LazyType]] # A symbol's ItemId. procInstCache*: Table[ItemId, seq[LazyInstantiation]] # A symbol's ItemId. @@ -81,7 +80,6 @@ type enumToStringProcs*: Table[ItemId, LazySym] emittedTypeInfo*: Table[string, FileIndex] - startupPackedConfig*: PackedConfig packageSyms*: TStrTable deps*: IntSet # the dependency graph or potentially its transitive closure. importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies @@ -213,79 +211,38 @@ proc strTableAdds*(g: ModuleGraph, m: PSym, s: PSym) = strTableAdd(semtab(g, m), s) strTableAdd(semtabAll(g, m), s) -proc isCachedModule(g: ModuleGraph; module: int): bool {.inline.} = - result = module < g.packed.len and g.packed[module].status == loaded - -proc isCachedModule*(g: ModuleGraph; m: PSym): bool {.inline.} = - isCachedModule(g, m.position) - -proc simulateCachedModule(g: ModuleGraph; moduleSym: PSym; m: PackedModule) = - when false: - echo "simulating ", moduleSym.name.s, " ", moduleSym.position - simulateLoadedModule(g.packed, g.config, g.cache, moduleSym, m) - type ModuleIter* = object - fromRod: bool modIndex: int ti: TIdentIter - rodIt: RodIter importHidden: bool proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym = assert m.kind == skModule mi.modIndex = m.position - mi.fromRod = isCachedModule(g, mi.modIndex) mi.importHidden = optImportHidden in m.options - if mi.fromRod: - result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name, mi.importHidden) - else: - result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name) + result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name) proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym = - if mi.fromRod: - result = nextRodIter(mi.rodIt, g.packed) - else: - result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden)) + result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden)) iterator allSyms*(g: ModuleGraph; m: PSym): PSym = let importHidden = optImportHidden in m.options - if isCachedModule(g, m): - var rodIt: RodIter = default(RodIter) - var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position, importHidden) - while r != nil: - yield r - r = nextRodIter(rodIt, g.packed) - else: - for s in g.ifaces[m.position].interfSelect(importHidden).data: - if s != nil: - yield s + for s in g.ifaces[m.position].interfSelect(importHidden).data: + if s != nil: + yield s proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym = let importHidden = optImportHidden in m.options - if isCachedModule(g, m): - result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden) - else: - result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name) + result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name) proc someSymAmb*(g: ModuleGraph; m: PSym; name: PIdent; amb: var bool): PSym = let importHidden = optImportHidden in m.options - if isCachedModule(g, m): - result = nil - for s in interfaceSymbols(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden): - if result == nil: - # set result to the first symbol - result = s - else: - # another symbol found - amb = true - break - else: - var ti: TIdentIter = default(TIdentIter) - result = initIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden), name) - if result != nil and nextIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden)) != nil: - # another symbol exists with same name - amb = true + var ti: TIdentIter = default(TIdentIter) + result = initIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden), name) + if result != nil and nextIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden)) != nil: + # another symbol exists with same name + amb = true proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym = result = someSym(g, g.systemModule, name) @@ -299,34 +256,18 @@ iterator systemModuleSyms*(g: ModuleGraph; name: PIdent): PSym = proc resolveType(g: ModuleGraph; t: var LazyType): PType = result = t.typ - if result == nil and isCachedModule(g, t.id.module): - result = loadTypeFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed) - t.typ = result assert result != nil proc resolveSym(g: ModuleGraph; t: var LazySym): PSym = result = t.sym - if result == nil and isCachedModule(g, t.id.module): - result = loadSymFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed) - t.sym = result assert result != nil proc resolveInst(g: ModuleGraph; t: var LazyInstantiation): PInstantiation = result = t.inst - if result == nil and isCachedModule(g, t.module): - result = PInstantiation(sym: loadSymFromId(g.config, g.cache, g.packed, t.sym.module, t.sym.packed)) - result.concreteTypes = newSeq[PType](t.concreteTypes.len) - for i in 0..high(result.concreteTypes): - result.concreteTypes[i] = loadTypeFromId(g.config, g.cache, g.packed, - t.concreteTypes[i].module, t.concreteTypes[i].packed) - t.inst = result assert result != nil proc resolveAttachedOp*(g: ModuleGraph; t: var LazySym): PSym = result = t.sym - if result == nil: - result = loadSymFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed) - t.sym = result assert result != nil iterator typeInstCacheItems*(g: ModuleGraph; s: PSym): PType = @@ -570,33 +511,14 @@ proc resetAllModules*(g: ModuleGraph) = initModuleGraphFields(g) proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym = - result = nil - if fileIdx.int32 >= 0: - if isCachedModule(g, fileIdx.int32): - result = g.packed[fileIdx.int32].module - elif fileIdx.int32 < g.ifaces.len: - result = g.ifaces[fileIdx.int32].module + if fileIdx.int32 >= 0 and fileIdx.int32 < g.ifaces.len: + result = g.ifaces[fileIdx.int32].module + else: + result = nil proc moduleOpenForCodegen*(g: ModuleGraph; m: FileIndex): bool {.inline.} = result = true -proc closeRodFile*(g: ModuleGraph; m: PSym) = - if g.config.symbolFiles in {readOnlySf, v2Sf}: - # For stress testing we seek to reload the symbols from memory. This - # way much of the logic is tested but the test is reproducible as it does - # not depend on the hard disk contents! - let mint = m.position - saveRodFile(toRodFile(g.config, AbsoluteFile toFullPath(g.config, FileIndex(mint))), - g.encoders[mint], g.packed[mint].fromDisk) - g.packed[mint].status = stored - - elif g.config.symbolFiles == stressTest: - # debug code, but maybe a good idea for production? Could reduce the compiler's - # memory consumption considerably at the cost of more loads from disk. - let mint = m.position - simulateCachedModule(g, m, g.packed[mint].fromDisk) - g.packed[mint].status = loaded - proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) = @@ -679,22 +601,8 @@ proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool = proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} = result = s.ast[bodyPos] - if result == nil and g.config.symbolFiles in {readOnlySf, v2Sf, stressTest}: - result = loadProcBody(g.config, g.cache, g.packed, s) - s.ast[bodyPos] = result assert result != nil -proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex; - cachedModules: var seq[FileIndex]): PSym = - ## Returns 'nil' if the module needs to be recompiled. - if g.config.symbolFiles in {readOnlySf, v2Sf, stressTest}: - result = moduleFromRodFile(g.packed, g.config, g.cache, fileIdx, cachedModules) - else: - result = nil - -proc configComplete*(g: ModuleGraph) = - rememberStartupConfig(g.startupPackedConfig, g.config) - proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, fromModule: PSym, ) = let conf = graph.config let isNimscript = conf.isDefined("nimscript") diff --git a/compiler/passes.nim b/compiler/passes.nim index 79fa629166b3..16a7f1f26875 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -147,11 +147,6 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; closeParser(p) if s.kind != llsStdIn: break closePasses(graph, a) - if graph.config.backend notin {backendC, backendCpp, backendObjc}: - # We only write rod files here if no C-like backend is active. - # The C-like backends have been patched to support the IC mechanism. - # They are responsible for closing the rod files. See `cbackend.nim`. - closeRodFile(graph, module) result = true proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym = @@ -167,20 +162,11 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fr elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput) discard processModule(graph, result, idGeneratorFromModule(result), s) if result == nil: - var cachedModules: seq[FileIndex] = @[] - result = moduleFromRodFile(graph, fileIdx, cachedModules) let filename = AbsoluteFile toFullPath(graph.config, fileIdx) - if result == nil: - result = newModule(graph, fileIdx) - result.flags.incl flags - registerModule(graph, result) - processModuleAux("import") - else: - if sfSystemModule in flags: - graph.systemModule = result - partialInitModule(result, graph, fileIdx, filename) - for m in cachedModules: - registerModuleById(graph, m) + result = newModule(graph, fileIdx) + result.flags.incl flags + registerModule(graph, result) + processModuleAux("import") elif graph.isDirty(result): result.flags.excl sfDirty # reset module fields: @@ -219,7 +205,6 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = connectCallbacks(graph) let conf = graph.config wantMainModule(conf) - configComplete(graph) let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim") let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 52b3a7f6da49..5b50d4898677 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -209,11 +209,6 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator of NonePass: raiseAssert "use setPipeLinePass to set a proper PipelinePass" - if graph.config.backend notin {backendC, backendCpp, backendObjc}: - # We only write rod files here if no C-like backend is active. - # The C-like backends have been patched to support the IC mechanism. - # They are responsible for closing the rod files. See `cbackend.nim`. - closeRodFile(graph, module) result = true proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags; fromModule: PSym = nil): PSym = @@ -229,27 +224,14 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput) discard processPipelineModule(graph, result, idGeneratorFromModule(result), s) if result == nil: - var cachedModules: seq[FileIndex] = @[] - result = moduleFromRodFile(graph, fileIdx, cachedModules) let path = toFullPath(graph.config, fileIdx) let filename = AbsoluteFile path if fileExists(filename): # it could be a stdinfile graph.cachedFiles[path] = $secureHashFile(path) - if result == nil: - result = newModule(graph, fileIdx) - result.flags.incl flags - registerModule(graph, result) - processModuleAux("import") - else: - if sfSystemModule in flags: - graph.systemModule = result - if sfMainModule in flags and graph.config.cmd == cmdM: - result.flags.incl flags - registerModule(graph, result) - processModuleAux("import") - partialInitModule(result, graph, fileIdx, filename) - for m in cachedModules: - registerModuleById(graph, m) + result = newModule(graph, fileIdx) + result.flags.incl flags + registerModule(graph, result) + processModuleAux("import") elif graph.isDirty(result): result.flags.excl sfDirty # reset module fields: @@ -288,7 +270,6 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx connectPipelineCallbacks(graph) let conf = graph.config wantMainModule(conf) - configComplete(graph) let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim") let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx From 217f43185c3e28d4811d5107589fa5a420cc24a2 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 30 Oct 2024 22:41:09 +0800 Subject: [PATCH 3/9] remove ic completely --- compiler/ic/bitabs.nim | 178 ----- compiler/ic/cbackend.nim | 180 ----- compiler/ic/dce.nim | 169 ----- compiler/ic/design.rst | 56 -- compiler/ic/ic.nim | 1343 ----------------------------------- compiler/ic/iclineinfos.nim | 84 --- compiler/ic/integrity.nim | 155 ---- compiler/ic/navigator.nim | 183 ----- compiler/ic/packed_ast.nim | 367 ---------- compiler/ic/replayer.nim | 171 ----- compiler/ic/rodfiles.nim | 283 -------- compiler/importer.nim | 15 +- compiler/modulegraphs.nim | 105 +-- compiler/semdata.nim | 22 +- compiler/semexprs.nim | 4 +- compiler/semstmts.nim | 2 +- compiler/semtempl.nim | 2 +- compiler/semtypes.nim | 2 +- compiler/vtables.nim | 18 +- 19 files changed, 54 insertions(+), 3285 deletions(-) delete mode 100644 compiler/ic/bitabs.nim delete mode 100644 compiler/ic/cbackend.nim delete mode 100644 compiler/ic/dce.nim delete mode 100644 compiler/ic/design.rst delete mode 100644 compiler/ic/ic.nim delete mode 100644 compiler/ic/iclineinfos.nim delete mode 100644 compiler/ic/integrity.nim delete mode 100644 compiler/ic/navigator.nim delete mode 100644 compiler/ic/packed_ast.nim delete mode 100644 compiler/ic/replayer.nim delete mode 100644 compiler/ic/rodfiles.nim diff --git a/compiler/ic/bitabs.nim b/compiler/ic/bitabs.nim deleted file mode 100644 index 0c9994c83ff6..000000000000 --- a/compiler/ic/bitabs.nim +++ /dev/null @@ -1,178 +0,0 @@ -## A BiTable is a table that can be seen as an optimized pair -## of `(Table[LitId, Val], Table[Val, LitId])`. - -import std/hashes -import rodfiles - -when defined(nimPreviewSlimSystem): - import std/assertions - -type - LitId* = distinct uint32 - - BiTable*[T] = object - vals: seq[T] # indexed by LitId - keys: seq[LitId] # indexed by hash(val) - -proc initBiTable*[T](): BiTable[T] = BiTable[T](vals: @[], keys: @[]) - -proc nextTry(h, maxHash: Hash): Hash {.inline.} = - result = (h + 1) and maxHash - -template maxHash(t): untyped = high(t.keys) -template isFilled(x: LitId): bool = x.uint32 > 0'u32 - -proc `$`*(x: LitId): string {.borrow.} -proc `<`*(x, y: LitId): bool {.borrow.} -proc `<=`*(x, y: LitId): bool {.borrow.} -proc `==`*(x, y: LitId): bool {.borrow.} -proc hash*(x: LitId): Hash {.borrow.} - - -proc len*[T](t: BiTable[T]): int = t.vals.len - -proc mustRehash(length, counter: int): bool {.inline.} = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) - -const - idStart = 1 - -template idToIdx(x: LitId): int = x.int - idStart - -proc hasLitId*[T](t: BiTable[T]; x: LitId): bool = - let idx = idToIdx(x) - result = idx >= 0 and idx < t.vals.len - -proc enlarge[T](t: var BiTable[T]) = - var n: seq[LitId] - newSeq(n, len(t.keys) * 2) - swap(t.keys, n) - for i in 0..high(n): - let eh = n[i] - if isFilled(eh): - var j = hash(t.vals[idToIdx eh]) and maxHash(t) - while isFilled(t.keys[j]): - j = nextTry(j, maxHash(t)) - t.keys[j] = move n[i] - -proc getKeyId*[T](t: BiTable[T]; v: T): LitId = - let origH = hash(v) - var h = origH and maxHash(t) - if t.keys.len != 0: - while true: - let litId = t.keys[h] - if not isFilled(litId): break - if t.vals[idToIdx t.keys[h]] == v: return litId - h = nextTry(h, maxHash(t)) - return LitId(0) - -proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId = - let origH = hash(v) - var h = origH and maxHash(t) - if t.keys.len != 0: - while true: - let litId = t.keys[h] - if not isFilled(litId): break - if t.vals[idToIdx t.keys[h]] == v: return litId - h = nextTry(h, maxHash(t)) - # not found, we need to insert it: - if mustRehash(t.keys.len, t.vals.len): - enlarge(t) - # recompute where to insert: - h = origH and maxHash(t) - while true: - let litId = t.keys[h] - if not isFilled(litId): break - h = nextTry(h, maxHash(t)) - else: - setLen(t.keys, 16) - h = origH and maxHash(t) - - result = LitId(t.vals.len + idStart) - t.keys[h] = result - t.vals.add v - - -proc `[]`*[T](t: var BiTable[T]; litId: LitId): var T {.inline.} = - let idx = idToIdx litId - assert idx < t.vals.len - result = t.vals[idx] - -proc `[]`*[T](t: BiTable[T]; litId: LitId): lent T {.inline.} = - let idx = idToIdx litId - assert idx < t.vals.len - result = t.vals[idx] - -proc hash*[T](t: BiTable[T]): Hash = - ## as the keys are hashes of the values, we simply use them instead - var h: Hash = 0 - for i, n in pairs t.keys: - h = h !& hash((i, n)) - result = !$h - -proc store*[T](f: var RodFile; t: BiTable[T]) = - storeSeq(f, t.vals) - storeSeq(f, t.keys) - -proc load*[T](f: var RodFile; t: var BiTable[T]) = - loadSeq(f, t.vals) - loadSeq(f, t.keys) - -proc sizeOnDisc*(t: BiTable[string]): int = - result = 4 - for x in t.vals: - result += x.len + 4 - result += t.keys.len * sizeof(LitId) - -when isMainModule: - - var t: BiTable[string] - - echo getOrIncl(t, "hello") - - echo getOrIncl(t, "hello") - echo getOrIncl(t, "hello3") - echo getOrIncl(t, "hello4") - echo getOrIncl(t, "helloasfasdfdsa") - echo getOrIncl(t, "hello") - echo getKeyId(t, "hello") - echo getKeyId(t, "none") - - for i in 0 ..< 100_000: - discard t.getOrIncl($i & "___" & $i) - - for i in 0 ..< 100_000: - assert t.getOrIncl($i & "___" & $i).idToIdx == i + 4 - echo "begin" - echo t.vals.len - - echo t.vals[0] - echo t.vals[1004] - - echo "middle" - - var tf: BiTable[float] - - discard tf.getOrIncl(0.4) - discard tf.getOrIncl(16.4) - discard tf.getOrIncl(32.4) - echo getKeyId(tf, 32.4) - - var f2 = open("testblah.bin", fmWrite) - echo store(f2, tf) - f2.close - - var f1 = open("testblah.bin", fmRead) - - var t2: BiTable[float] - - echo f1.load(t2) - echo t2.vals.len - - echo getKeyId(t2, 32.4) - - echo "end" - - - f1.close diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim deleted file mode 100644 index 83f1b4cc75bd..000000000000 --- a/compiler/ic/cbackend.nim +++ /dev/null @@ -1,180 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2021 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## New entry point into our C/C++ code generator. Ideally -## somebody would rewrite the old backend (which is 8000 lines of crufty Nim code) -## to work on packed trees directly and produce the C code as an AST which can -## then be rendered to text in a very simple manner. Unfortunately nobody wrote -## this code. So instead we wrap the existing cgen.nim and its friends so that -## we call directly into the existing code generation logic but avoiding the -## naive, outdated `passes` design. Thus you will see some -## `useAliveDataFromDce in flags` checks in the old code -- the old code is -## also doing cross-module dependency tracking and DCE that we don't need -## anymore. DCE is now done as prepass over the entire packed module graph. - -import std/[packedsets, algorithm, tables] - -when defined(nimPreviewSlimSystem): - import std/assertions - -import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen, - pathutils, extccomp, msgs, modulepaths] - -import packed_ast, ic, dce, rodfiles - -proc unpackTree(g: ModuleGraph; thisModule: int; - tree: PackedTree; n: NodePos): PNode = - var decoder = initPackedDecoder(g.config, g.cache) - result = loadNodes(decoder, g.packed, thisModule, tree, n) - -proc setupBackendModule(g: ModuleGraph; m: var LoadedModule) = - if g.backend == nil: - g.backend = cgendata.newModuleList(g) - assert g.backend != nil - var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config) - bmod.idgen = idgenFromLoadedModule(m) - -proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) = - var bmod = BModuleList(g.backend).modules[m.module.position] - assert bmod != nil - bmod.flags.incl useAliveDataFromDce - bmod.alive = move alive[m.module.position] - - for p in allNodes(m.fromDisk.topLevel): - let n = unpackTree(g, m.module.position, m.fromDisk.topLevel, p) - cgen.genTopLevelStmt(bmod, n) - - finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info)) - for disp in getDispatchers(g): - genProcAux(bmod, disp) - m.fromDisk.backendFlags = cgen.whichInitProcs(bmod) - -proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) = - for x in mitems(m.fromDisk.emittedTypeInfo): - #echo "found type ", x, " for file ", int(origin) - g.emittedTypeInfo[x] = origin - -proc addFileToLink(config: ConfigRef; m: PSym) = - let filename = AbsoluteFile toFullPath(config, m.position.FileIndex) - let ext = - if config.backend == backendCpp: ".nim.cpp" - elif config.backend == backendObjc: ".nim.m" - else: ".nim.c" - let cfile = changeFileExt(completeCfilePath(config, - mangleModuleName(config, filename).AbsoluteFile), ext) - let objFile = completeCfilePath(config, toObjFile(config, cfile)) - if fileExists(objFile): - var cf = Cfile(nimname: m.name.s, cname: cfile, - obj: objFile, - flags: {CfileFlag.Cached}) - addFileToCompile(config, cf) - -when defined(debugDce): - import os, std/packedsets - -proc storeAliveSymsImpl(asymFile: AbsoluteFile; s: seq[int32]) = - var f = rodfiles.create(asymFile.string) - f.storeHeader() - f.storeSection aliveSymsSection - f.storeSeq(s) - close f - -template prepare {.dirty.} = - let asymFile = toRodFile(config, AbsoluteFile toFullPath(config, position.FileIndex), ".alivesyms") - var s = newSeqOfCap[int32](alive[position].len) - for a in items(alive[position]): s.add int32(a) - sort(s) - -proc storeAliveSyms(config: ConfigRef; position: int; alive: AliveSyms) = - prepare() - storeAliveSymsImpl(asymFile, s) - -proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool = - prepare() - var f2 = rodfiles.open(asymFile.string) - f2.loadHeader() - f2.loadSection aliveSymsSection - var oldData: seq[int32] = @[] - f2.loadSeq(oldData) - f2.close - if f2.err == ok and oldData == s: - result = false - else: - when defined(debugDce): - let oldAsSet = toPackedSet[int32](oldData) - let newAsSet = toPackedSet[int32](s) - echo "set of live symbols changed ", asymFile.changeFileExt("rod"), " ", position, " ", f2.err - echo "in old but not in new ", oldAsSet.difference(newAsSet), " number of entries in old ", oldAsSet.len - echo "in new but not in old ", newAsSet.difference(oldAsSet), " number of entries in new ", newAsSet.len - #if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0: - # echo "command failed" - result = true - storeAliveSymsImpl(asymFile, s) - -proc genPackedModule(g: ModuleGraph, i: int; alive: var AliveSyms) = - # case statement here to enforce exhaustive checks. - case g.packed[i].status - of undefined: - discard "nothing to do" - of loading, stored: - assert false - of storing, outdated: - storeAliveSyms(g.config, g.packed[i].module.position, alive) - generateCodeForModule(g, g.packed[i], alive) - closeRodFile(g, g.packed[i].module) - of loaded: - if g.packed[i].loadedButAliveSetChanged: - generateCodeForModule(g, g.packed[i], alive) - else: - addFileToLink(g.config, g.packed[i].module) - replayTypeInfo(g, g.packed[i], FileIndex(i)) - - if g.backend == nil: - g.backend = cgendata.newModuleList(g) - registerInitProcs(BModuleList(g.backend), g.packed[i].module, g.packed[i].fromDisk.backendFlags) - -proc generateCode*(g: ModuleGraph) = - ## The single entry point, generate C(++) code for the entire - ## Nim program aka `ModuleGraph`. - resetForBackend(g) - var alive = computeAliveSyms(g.packed, g.config) - - when false: - for i in 0..= 0: - genPackedModule(g, mainModuleIdx, alive) diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim deleted file mode 100644 index 6eb36431ea04..000000000000 --- a/compiler/ic/dce.nim +++ /dev/null @@ -1,169 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2021 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Dead code elimination (=DCE) for IC. - -import std/[intsets, tables] - -when defined(nimPreviewSlimSystem): - import std/assertions - -import ".." / [ast, options, lineinfos, types] - -import packed_ast, ic, bitabs - -type - AliveSyms* = seq[IntSet] - AliveContext* = object ## Purpose is to fill the 'alive' field. - stack: seq[(int, TOptions, NodePos)] ## A stack for marking symbols as alive. - decoder: PackedDecoder ## We need a PackedDecoder for module ID address translations. - thisModule: int ## The module we're currently analysing for DCE. - alive: AliveSyms ## The final result of our computation. - options: TOptions - compilerProcs: Table[string, (int, int32)] - -proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): bool = - ## "Exported to C" procs are special (these are marked with '.exportc') because these - ## must not be optimized away! - let symPtr = unsafeAddr g[c.thisModule].fromDisk.syms[symId] - let flags = symPtr.flags - # due to a bug/limitation in the lambda lifting, unused inner procs - # are not transformed correctly; issue (#411). However, the whole purpose here - # is to eliminate unused procs. So there is no special logic required for this case. - if sfCompileTime notin flags: - if ({sfExportc, sfCompilerProc} * flags != {}) or - (symPtr.kind == skMethod): - result = true - else: - result = false - # XXX: This used to be a condition to: - # (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or - if sfCompilerProc in flags: - c.compilerProcs[g[c.thisModule].fromDisk.strings[symPtr.name]] = (c.thisModule, symId) - else: - result = false - -template isNotGeneric(n: NodePos): bool = ithSon(tree, n, genericParamsPos).kind == nkEmpty - -proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: int32) = - ## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will - ## be analysed. - if not c.alive[module].containsOrIncl(item): - var body = g[module].fromDisk.syms[item].ast - if body != emptyNodeId: - let opt = g[module].fromDisk.syms[item].options - if g[module].fromDisk.syms[item].kind in routineKinds: - body = NodeId ithSon(g[module].fromDisk.bodies, NodePos body, bodyPos) - c.stack.add((module, opt, NodePos(body))) - - when false: - let nid = g[module].fromDisk.syms[item].name - if nid != LitId(0): - let name = g[module].fromDisk.strings[nid] - if name in ["nimFrame", "callDepthLimitReached"]: - echo "I was called! ", name, " body exists: ", body != emptyNodeId, " ", module, " ", item - -proc requestCompilerProc(c: var AliveContext; g: PackedModuleGraph; name: string) = - let (module, item) = c.compilerProcs[name] - followLater(c, g, module, item) - -proc loadTypeKind(t: PackedItemId; c: AliveContext; g: PackedModuleGraph; toSkip: set[TTypeKind]): TTypeKind = - template kind(t: ItemId): TTypeKind = g[t.module].fromDisk.types[t.item].kind - - var t2 = translateId(t, g, c.thisModule, c.decoder.config) - result = t2.kind - while result in toSkip: - t2 = translateId(g[t2.module].fromDisk.types[t2.item].types[^1], g, t2.module, c.decoder.config) - result = t2.kind - -proc rangeCheckAnalysis(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) = - ## Replicates the logic of `ccgexprs.genRangeChck`. - ## XXX Refactor so that the duplicated logic is avoided. However, for now it's not clear - ## the approach has enough merit. - var dest = loadTypeKind(n.typ, c, g, abstractVar) - if optRangeCheck notin c.options or dest in {tyUInt..tyUInt64}: - discard "no need to generate a check because it was disabled" - else: - let n0t = loadTypeKind(n.firstSon.typ, c, g, {}) - if n0t in {tyUInt, tyUInt64}: - c.requestCompilerProc(g, "raiseRangeErrorNoArgs") - else: - let raiser = - case loadTypeKind(n.typ, c, g, abstractVarRange) - of tyUInt..tyUInt64, tyChar: "raiseRangeErrorU" - of tyFloat..tyFloat128: "raiseRangeErrorF" - else: "raiseRangeErrorI" - c.requestCompilerProc(g, raiser) - -proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) = - ## Marks the symbols we encounter when we traverse the AST at `tree[n]` as alive, unless - ## it is purely in a declarative context (type section etc.). - case n.kind - of nkNone..pred(nkSym), succ(nkSym)..nkNilLit: - discard "ignore non-sym atoms" - of nkSym: - # This symbol is alive and everything its body references. - followLater(c, g, c.thisModule, tree[n].soperand) - of nkModuleRef: - let (n1, n2) = sons2(tree, n) - assert n1.kind == nkNone - assert n2.kind == nkNone - let m = n1.litId - let item = tree[n2].soperand - let otherModule = toFileIndexCached(c.decoder, g, c.thisModule, m).int - followLater(c, g, otherModule, item) - of nkMacroDef, nkTemplateDef, nkTypeSection, nkTypeOfExpr, - nkCommentStmt, nkIncludeStmt, - nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt, - nkFromStmt, nkStaticStmt: - discard - of nkVarSection, nkLetSection, nkConstSection: - # XXX ignore the defining local variable name? - for son in sonsReadonly(tree, n): - aliveCode(c, g, tree, son) - of nkChckRangeF, nkChckRange64, nkChckRange: - rangeCheckAnalysis(c, g, tree, n) - of nkProcDef, nkConverterDef, nkMethodDef, nkFuncDef, nkIteratorDef: - if n.firstSon.kind == nkSym and isNotGeneric(n): - let item = tree[n.firstSon].soperand - if isExportedToC(c, g, item): - # This symbol is alive and everything its body references. - followLater(c, g, c.thisModule, item) - else: - for son in sonsReadonly(tree, n): - aliveCode(c, g, tree, son) - -proc followNow(c: var AliveContext; g: PackedModuleGraph) = - ## Mark all entries in the stack. Marking can add more entries - ## to the stack but eventually we have looked at every alive symbol. - while c.stack.len > 0: - let (modId, opt, ast) = c.stack.pop() - c.thisModule = modId - c.options = opt - aliveCode(c, g, g[modId].fromDisk.bodies, ast) - -proc computeAliveSyms*(g: PackedModuleGraph; conf: ConfigRef): AliveSyms = - ## Entry point for our DCE algorithm. - var c = AliveContext(stack: @[], decoder: PackedDecoder(config: conf), - thisModule: -1, alive: newSeq[IntSet](g.len), - options: conf.options) - for i in countdown(len(g)-1, 0): - if g[i].status != undefined: - c.thisModule = i - for p in allNodes(g[i].fromDisk.topLevel): - aliveCode(c, g, g[i].fromDisk.topLevel, p) - - followNow(c, g) - result = move(c.alive) - -proc isAlive*(a: AliveSyms; module: int, item: int32): bool = - ## Backends use this to query if a symbol is `alive` which means - ## we need to produce (C/C++/etc) code for it. - result = a[module].contains(item) - diff --git a/compiler/ic/design.rst b/compiler/ic/design.rst deleted file mode 100644 index b096e3103a17..000000000000 --- a/compiler/ic/design.rst +++ /dev/null @@ -1,56 +0,0 @@ -==================================== - Incremental Recompilations -==================================== - -We split the Nim compiler into a frontend and a backend. -The frontend produces a set of `.rod` files. Every `.nim` module -produces its own `.rod` file. - -- The IR must be a faithful representation of the AST in memory. -- The backend can do its own caching but doesn't have to. In the - current implementation the backend also caches its results. - -Advantage of the "set of files" vs the previous global database: -- By construction, we either read from the `.rod` file or from the - `.nim` file, there can be no inconsistency. There can also be no - partial updates. -- No dependency to external packages (SQLite). SQLite simply is too - slow and the old way of serialization was too slow too. We use a - format designed for Nim and expect to base further tools on this - file format. - -References to external modules must be (moduleId, symId) pairs. -The symbol IDs are module specific. This way no global ID increment -mechanism needs to be implemented that we could get wrong. ModuleIds -are rod-file specific too. - - - -Global state ------------- - -There is no global state. - -Rod File Format ---------------- - -It's a simple binary file format. `rodfiles.nim` contains some details. - - -Backend -------- - -Nim programmers have to come to enjoy whole-program dead code elimination, -by default. Since this is a "whole program" optimization, it does break -modularity. However, thanks to the packed AST representation we can perform -this global analysis without having to unpack anything. This is basically -a mark&sweep GC algorithm: - -- Start with the top level statements. Every symbol that is referenced - from a top level statement is not "dead" and needs to be compiled by - the backend. -- Every symbol referenced from a referenced symbol also has to be - compiled. - -Caching logic: Only if the set of alive symbols is different from the -last run, the module has to be regenerated. diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim deleted file mode 100644 index ecc6069e7543..000000000000 --- a/compiler/ic/ic.nim +++ /dev/null @@ -1,1343 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2020 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import std/[hashes, tables, intsets, monotimes] -import packed_ast, bitabs, rodfiles -import ".." / [ast, idents, lineinfos, msgs, ropes, options, - pathutils, condsyms, packages, modulepaths] -#import ".." / [renderer, astalgo] -from std/os import removeFile, isAbsolute - -import ../../dist/checksums/src/checksums/sha1 - -import iclineinfos - -when defined(nimPreviewSlimSystem): - import std/[syncio, assertions, formatfloat] - -type - PackedConfig* = object - backend: TBackend - selectedGC: TGCMode - cCompiler: TSystemCC - options: TOptions - globalOptions: TGlobalOptions - - ModuleBackendFlag* = enum - HasDatInitProc - HasModuleInitProc - - PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file - definedSymbols: string - moduleFlags: TSymFlags - includes*: seq[(LitId, string)] # first entry is the module filename itself - imports: seq[LitId] # the modules this module depends on - toReplay*: PackedTree # pragmas and VM specific state to replay. - topLevel*: PackedTree # top level statements - bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position. - #producedGenerics*: Table[GenericKey, SymId] - exports*: seq[(LitId, int32)] - hidden: seq[(LitId, int32)] - reexports: seq[(LitId, PackedItemId)] - compilerProcs*: seq[(LitId, int32)] - converters*, methods*, trmacros*, pureEnums*: seq[int32] - - typeInstCache*: seq[(PackedItemId, PackedItemId)] - procInstCache*: seq[PackedInstantiation] - attachedOps*: seq[(PackedItemId, TTypeAttachedOp, PackedItemId)] - methodsPerGenericType*: seq[(PackedItemId, int, PackedItemId)] - enumToStringProcs*: seq[(PackedItemId, PackedItemId)] - methodsPerType*: seq[(PackedItemId, PackedItemId)] - dispatchers*: seq[PackedItemId] - - emittedTypeInfo*: seq[string] - backendFlags*: set[ModuleBackendFlag] - - syms*: OrderedTable[int32, PackedSym] - types*: OrderedTable[int32, PackedType] - strings*: BiTable[string] # we could share these between modules. - numbers*: BiTable[BiggestInt] # we also store floats in here so - # that we can assure that every bit is kept - man*: LineInfoManager - - cfg: PackedConfig - - PackedEncoder* = object - #m*: PackedModule - thisModule*: int32 - lastFile*: FileIndex # remember the last lookup entry. - lastLit*: LitId - filenames*: Table[FileIndex, LitId] - pendingTypes*: seq[PType] - pendingSyms*: seq[PSym] - typeMarker*: IntSet #Table[ItemId, TypeId] # ItemId.item -> TypeId - symMarker*: IntSet #Table[ItemId, SymId] # ItemId.item -> SymId - config*: ConfigRef - -proc toString*(tree: PackedTree; pos: NodePos; m: PackedModule; nesting: int; - result: var string) = - if result.len > 0 and result[^1] notin {' ', '\n'}: - result.add ' ' - - result.add $tree[pos].kind - case tree[pos].kind - of nkEmpty, nkNilLit, nkType: discard - of nkIdent, nkStrLit..nkTripleStrLit: - result.add " " - result.add m.strings[LitId tree[pos].uoperand] - of nkSym: - result.add " " - result.add m.strings[m.syms[tree[pos].soperand].name] - of directIntLit: - result.add " " - result.addInt tree[pos].soperand - of externSIntLit: - result.add " " - result.addInt m.numbers[LitId tree[pos].uoperand] - of externUIntLit: - result.add " " - result.addInt cast[uint64](m.numbers[LitId tree[pos].uoperand]) - of nkFloatLit..nkFloat128Lit: - result.add " " - result.addFloat cast[BiggestFloat](m.numbers[LitId tree[pos].uoperand]) - else: - result.add "(\n" - for i in 1..(nesting+1)*2: result.add ' ' - for child in sonsReadonly(tree, pos): - toString(tree, child, m, nesting + 1, result) - result.add "\n" - for i in 1..nesting*2: result.add ' ' - result.add ")" - #for i in 1..nesting*2: result.add ' ' - -proc toString*(tree: PackedTree; n: NodePos; m: PackedModule): string = - result = "" - toString(tree, n, m, 0, result) - -proc debug*(tree: PackedTree; m: PackedModule) = - stdout.write toString(tree, NodePos 0, m) - -proc isActive*(e: PackedEncoder): bool = e.config != nil -proc disable(e: var PackedEncoder) = e.config = nil - -template primConfigFields(fn: untyped) {.dirty.} = - fn backend - fn selectedGC - fn cCompiler - fn options - fn globalOptions - -proc definedSymbolsAsString(config: ConfigRef): string = - result = newStringOfCap(200) - result.add "config" - for d in definedSymbolNames(config.symbols): - result.add ' ' - result.add d - -proc rememberConfig(c: var PackedEncoder; m: var PackedModule; config: ConfigRef; pc: PackedConfig) = - m.definedSymbols = definedSymbolsAsString(config) - #template rem(x) = - # c.m.cfg.x = config.x - #primConfigFields rem - m.cfg = pc - -const - debugConfigDiff = defined(debugConfigDiff) - -when debugConfigDiff: - import hashes, tables, intsets, sha1, strutils, sets - -proc configIdentical(m: PackedModule; config: ConfigRef): bool = - result = m.definedSymbols == definedSymbolsAsString(config) - when debugConfigDiff: - if not result: - var wordsA = m.definedSymbols.split(Whitespace).toHashSet() - var wordsB = definedSymbolsAsString(config).split(Whitespace).toHashSet() - for c in wordsA - wordsB: - echo "in A but not in B ", c - for c in wordsB - wordsA: - echo "in B but not in A ", c - template eq(x) = - result = result and m.cfg.x == config.x - when debugConfigDiff: - if m.cfg.x != config.x: - echo "B ", m.cfg.x, " ", config.x - primConfigFields eq - -proc rememberStartupConfig*(dest: var PackedConfig, config: ConfigRef) = - template rem(x) = - dest.x = config.x - primConfigFields rem - dest.globalOptions.excl optForceFullMake - -proc hashFileCached(conf: ConfigRef; fileIdx: FileIndex): string = - result = msgs.getHash(conf, fileIdx) - if result.len == 0: - let fullpath = msgs.toFullPath(conf, fileIdx) - result = $secureHashFile(fullpath) - msgs.setHash(conf, fileIdx, result) - -proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId = - ## store a file index as a literal - if x == c.lastFile: - result = c.lastLit - else: - result = c.filenames.getOrDefault(x) - if result == LitId(0): - let p = msgs.toFullPath(c.config, x) - result = getOrIncl(m.strings, p) - c.filenames[x] = result - c.lastFile = x - c.lastLit = result - assert result != LitId(0) - -proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex = - result = msgs.fileInfoIdx(config, AbsoluteFile m.strings[x]) - -proc includesIdentical(m: var PackedModule; config: ConfigRef): bool = - for it in mitems(m.includes): - if hashFileCached(config, toFileIndex(it[0], m, config)) != it[1]: - return false - result = true - -proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; config: ConfigRef; pc: PackedConfig) = - ## setup a context for serializing to packed ast - c.thisModule = moduleSym.itemId.module - c.config = config - m.moduleFlags = moduleSym.flags - m.bodies = newTreeFrom(m.topLevel) - m.toReplay = newTreeFrom(m.topLevel) - - c.lastFile = FileIndex(-10) - - let thisNimFile = FileIndex c.thisModule - var h = msgs.getHash(config, thisNimFile) - if h.len == 0: - let fullpath = msgs.toFullPath(config, thisNimFile) - if isAbsolute(fullpath): - # For NimScript compiler API support the main Nim file might be from a stream. - h = $secureHashFile(fullpath) - msgs.setHash(config, thisNimFile, h) - m.includes.add((toLitId(thisNimFile, c, m), h)) # the module itself - - rememberConfig(c, m, config, pc) - -proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) = - m.includes.add((toLitId(f, c, m), hashFileCached(c.config, f))) - -proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) = - m.imports.add toLitId(f, c, m) - -proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - assert s.kind != skUnknown - let nameId = getOrIncl(m.strings, s.name.s) - m.hidden.add((nameId, s.itemId.item)) - assert s.itemId.module == c.thisModule - -proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - assert s.kind != skUnknown - assert s.itemId.module == c.thisModule - let nameId = getOrIncl(m.strings, s.name.s) - m.exports.add((nameId, s.itemId.item)) - -proc addConverter*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - assert c.thisModule == s.itemId.module - m.converters.add(s.itemId.item) - -proc addTrmacro*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - m.trmacros.add(s.itemId.item) - -proc addPureEnum*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - assert s.kind == skType - m.pureEnums.add(s.itemId.item) - -proc addMethod*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - m.methods.add s.itemId.item - -proc addReexport*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - assert s.kind != skUnknown - if s.kind == skModule: return - let nameId = getOrIncl(m.strings, s.name.s) - m.reexports.add((nameId, PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), - item: s.itemId.item))) - -proc addCompilerProc*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - let nameId = getOrIncl(m.strings, s.name.s) - m.compilerProcs.add((nameId, s.itemId.item)) - -proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) -proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId -proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId - -proc flush(c: var PackedEncoder; m: var PackedModule) = - ## serialize any pending types or symbols from the context - while true: - if c.pendingTypes.len > 0: - discard storeType(c.pendingTypes.pop, c, m) - elif c.pendingSyms.len > 0: - discard storeSym(c.pendingSyms.pop, c, m) - else: - break - -proc toLitId(x: string; m: var PackedModule): LitId = - ## store a string as a literal - result = getOrIncl(m.strings, x) - -proc toLitId(x: BiggestInt; m: var PackedModule): LitId = - ## store an integer as a literal - result = getOrIncl(m.numbers, x) - -proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo = - pack(m.man, toLitId(x.fileIndex, c, m), x.line.int32, x.col.int32) - #PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m)) - -proc safeItemId(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId {.inline.} = - ## given a symbol, produce an ItemId with the correct properties - ## for local or remote symbols, packing the symbol as necessary - if s == nil or s.kind == skPackage: - result = nilItemId - #elif s.itemId.module == c.thisModule: - # result = PackedItemId(module: LitId(0), item: s.itemId.item) - else: - assert int(s.itemId.module) >= 0 - result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), - item: s.itemId.item) - -proc addMissing(c: var PackedEncoder; p: PSym) = - ## consider queuing a symbol for later addition to the packed tree - if p != nil and p.itemId.module == c.thisModule: - if p.itemId.item notin c.symMarker: - if not (sfForward in p.flags and p.kind in routineKinds): - c.pendingSyms.add p - -proc addMissing(c: var PackedEncoder; p: PType) = - ## consider queuing a type for later addition to the packed tree - if p != nil and p.uniqueId.module == c.thisModule: - if p.uniqueId.item notin c.typeMarker: - c.pendingTypes.add p - -template storeNode(dest, src, field) = - var nodeId: NodeId - if src.field != nil: - nodeId = getNodeId(m.bodies) - toPackedNode(src.field, m.bodies, c, m) - else: - nodeId = emptyNodeId - dest.field = nodeId - -proc storeTypeLater(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId = - # We store multiple different trees in m.bodies. For this to work out, we - # cannot immediately store types/syms. We enqueue them instead to ensure - # we only write one tree into m.bodies after the other. - if t.isNil: return nilItemId - - assert t.uniqueId.module >= 0 - assert t.uniqueId.item > 0 - result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) - if t.uniqueId.module == c.thisModule: - # the type belongs to this module, so serialize it here, eventually. - addMissing(c, t) - -proc storeSymLater(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId = - if s.isNil: return nilItemId - assert s.itemId.module >= 0 - assert s.itemId.item >= 0 - result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - if s.itemId.module == c.thisModule: - # the sym belongs to this module, so serialize it here, eventually. - addMissing(c, s) - -proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId = - ## serialize a ptype - if t.isNil: return nilItemId - - assert t.uniqueId.module >= 0 - assert t.uniqueId.item > 0 - result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) - - if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item): - #if t.uniqueId.item >= m.types.len: - # setLen m.types, t.uniqueId.item+1 - - var p = PackedType(id: t.uniqueId.item, kind: t.kind, flags: t.flags, callConv: t.callConv, - size: t.size, align: t.align, nonUniqueId: t.itemId.item, - paddingAtEnd: t.paddingAtEnd) - storeNode(p, t, n) - p.typeInst = t.typeInst.storeType(c, m) - for kid in kids t: - p.types.add kid.storeType(c, m) - c.addMissing t.sym - p.sym = t.sym.safeItemId(c, m) - c.addMissing t.owner - p.owner = t.owner.safeItemId(c, m) - - # fill the reserved slot, nothing else: - m.types[t.uniqueId.item] = p - -proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib = - ## the plib hangs off the psym via the .annex field - if l.isNil: return - result = PackedLib(kind: l.kind, generated: l.generated, - isOverridden: l.isOverridden, name: toLitId($l.name, m) - ) - storeNode(result, l, path) - -proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId = - ## serialize a psym - if s.isNil: return nilItemId - - assert s.itemId.module >= 0 - result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - - if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item): - #if s.itemId.item >= m.syms.len: - # setLen m.syms, s.itemId.item+1 - - assert sfForward notin s.flags - - var p = PackedSym(id: s.itemId.item, kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic, - position: s.position, offset: s.offset, disamb: s.disamb, options: s.options, - name: s.name.s.toLitId(m)) - - storeNode(p, s, ast) - storeNode(p, s, constraint) - - if s.kind in {skLet, skVar, skField, skForVar}: - c.addMissing s.guard - p.guard = s.guard.safeItemId(c, m) - p.bitsize = s.bitsize - p.alignment = s.alignment - - p.externalName = toLitId(s.loc.snippet, m) - p.locFlags = s.loc.flags - c.addMissing s.typ - p.typ = s.typ.storeType(c, m) - c.addMissing s.owner - p.owner = s.owner.safeItemId(c, m) - p.annex = toPackedLib(s.annex, c, m) - when hasFFI: - p.cname = toLitId(s.cname, m) - p.instantiatedFrom = s.instantiatedFrom.safeItemId(c, m) - - # fill the reserved slot, nothing else: - m.syms[s.itemId.item] = p - -proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = - ## add a remote symbol reference to the tree - let info = n.info.toPackedInfo(c, m) - if n.typ != n.sym.typ: - ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total - info = info, flags = n.flags, - typeId = storeTypeLater(n.typ, c, m)) - else: - ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total - info = info, flags = n.flags) - ir.addNode(kind = nkNone, info = info, - operand = toLitId(n.sym.itemId.module.FileIndex, c, m).int32) - ir.addNode(kind = nkNone, info = info, - operand = n.sym.itemId.item) - -proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = - ## serialize a node into the tree - if n == nil: - ir.addNode(kind = nkNilRodNode, operand = 1, info = NoLineInfo) - return - let info = toPackedInfo(n.info, c, m) - case n.kind - of nkNone, nkEmpty, nkNilLit, nkType: - ir.addNode(kind = n.kind, flags = n.flags, operand = 0, - typeId = storeTypeLater(n.typ, c, m), info = info) - of nkIdent: - ir.addNode(kind = n.kind, flags = n.flags, - operand = int32 getOrIncl(m.strings, n.ident.s), - typeId = storeTypeLater(n.typ, c, m), info = info) - of nkSym: - if n.sym.itemId.module == c.thisModule: - # it is a symbol that belongs to the module we're currently - # packing: - let id = n.sym.storeSymLater(c, m).item - if n.typ != n.sym.typ: - ir.addNode(kind = nkSym, flags = n.flags, operand = id, - info = info, - typeId = storeTypeLater(n.typ, c, m)) - else: - ir.addNode(kind = nkSym, flags = n.flags, operand = id, - info = info) - else: - # store it as an external module reference: - addModuleRef(n, ir, c, m) - of externIntLit: - ir.addNode(kind = n.kind, flags = n.flags, - operand = int32 getOrIncl(m.numbers, n.intVal), - typeId = storeTypeLater(n.typ, c, m), info = info) - of nkStrLit..nkTripleStrLit: - ir.addNode(kind = n.kind, flags = n.flags, - operand = int32 getOrIncl(m.strings, n.strVal), - typeId = storeTypeLater(n.typ, c, m), info = info) - of nkFloatLit..nkFloat128Lit: - ir.addNode(kind = n.kind, flags = n.flags, - operand = int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)), - typeId = storeTypeLater(n.typ, c, m), info = info) - else: - let patchPos = ir.prepare(n.kind, n.flags, - storeTypeLater(n.typ, c, m), info) - for i in 0..= g.len: - g.pm.setLen(si+1) - - if g[si].status == undefined and c.config.cmd == cmdM: - var cachedModules: seq[FileIndex] = @[] - discard needsRecompile(g, c.config, c.cache, FileIndex(si), cachedModules) - for m in cachedModules: - loadToReplayNodes(g, c.config, c.cache, m, g[int m]) - - assert g[si].status in {loaded, storing, stored} - #if not g[si].symsInit: - # g[si].symsInit = true - # setLen g[si].syms, g[si].fromDisk.syms.len - - if g[si].syms.getOrDefault(s.item) == nil: - if g[si].fromDisk.syms[s.item].kind != skModule: - result = symHeaderFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item) - # store it here early on, so that recursions work properly: - g[si].syms[s.item] = result - symBodyFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item, result) - else: - result = g[si].module - assert result != nil - g[si].syms[s.item] = result - - else: - result = g[si].syms[s.item] - -proc typeHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; - t: PackedType; si, item: int32): PType = - result = PType(itemId: ItemId(module: si, item: t.nonUniqueId), kind: t.kind, - flags: t.flags, size: t.size, align: t.align, - paddingAtEnd: t.paddingAtEnd, - uniqueId: ItemId(module: si, item: item), - callConv: t.callConv) - -proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; - t: PackedType; si, item: int32; result: PType) = - result.sym = loadSym(c, g, si, t.sym) - setOwner(result, loadSym(c, g, si, t.owner)) - when false: - for op, item in pairs t.attachedOps: - result.attachedOps[op] = loadSym(c, g, si, item) - result.typeInst = loadType(c, g, si, t.typeInst) - var sons = newSeq[PType]() - for son in items t.types: - sons.add loadType(c, g, si, son) - result.setSons(sons) - loadAstBody(t, n) - when false: - for gen, id in items t.methods: - result.methods.add((gen, loadSym(c, g, si, id))) - -proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t: PackedItemId): PType = - if t == nilItemId: - result = nil - else: - let si = moduleIndex(c, g, thisModule, t) - assert g[si].status in {loaded, storing, stored} - assert t.item > 0 - - #if not g[si].typesInit: - # g[si].typesInit = true - # setLen g[si].types, g[si].fromDisk.types.len - - if g[si].types.getOrDefault(t.item) == nil: - result = typeHeaderFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item) - # store it here early on, so that recursions work properly: - g[si].types[t.item] = result - typeBodyFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item, result) - #assert result.itemId.item == t.item, $(result.itemId.item, t.item) - assert result.itemId.item > 0, $(result.itemId.item, t.item) - else: - result = g[si].types[t.item] - assert result.itemId.item > 0, "2" - -proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; - fileIdx: FileIndex; m: var LoadedModule) = - m.iface = initTable[PIdent, seq[PackedItemId]]() - m.ifaceHidden = initTable[PIdent, seq[PackedItemId]]() - template impl(iface, e) = - let nameLit = e[0] - let e2 = - when e[1] is PackedItemId: e[1] - else: PackedItemId(module: LitId(0), item: e[1]) - iface.mgetOrPut(cache.getIdent(m.fromDisk.strings[nameLit]), @[]).add(e2) - - for e in m.fromDisk.exports: - m.iface.impl(e) - m.ifaceHidden.impl(e) - for e in m.fromDisk.reexports: - m.iface.impl(e) - m.ifaceHidden.impl(e) - for e in m.fromDisk.hidden: - m.ifaceHidden.impl(e) - - let filename = AbsoluteFile toFullPath(conf, fileIdx) - # We cannot call ``newSym`` here, because we have to circumvent the ID - # mechanism, which we do in order to assign each module a persistent ID. - m.module = PSym(kind: skModule, itemId: ItemId(module: int32(fileIdx), item: 0'i32), - name: getIdent(cache, splitFile(filename).name), - info: newLineInfo(fileIdx, 1, 1), - position: int(fileIdx)) - setOwner(m.module, getPackage(conf, cache, fileIdx)) - m.module.flags = m.fromDisk.moduleFlags - -proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; - fileIdx: FileIndex; m: var LoadedModule) = - m.module.ast = newNode(nkStmtList) - if m.fromDisk.toReplay.len > 0: - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: conf, - cache: cache) - for p in allNodes(m.fromDisk.toReplay): - m.module.ast.add loadNodes(decoder, g, int(fileIdx), m.fromDisk.toReplay, p) - -proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; - fileIdx: FileIndex; cachedModules: var seq[FileIndex]): bool = - # Does the file belong to the fileIdx need to be recompiled? - let m = int(fileIdx) - if m >= g.len: - g.pm.setLen(m+1) - - case g[m].status - of undefined: - g[m].status = loading - let fullpath = msgs.toFullPath(conf, fileIdx) - let rod = toRodFile(conf, AbsoluteFile fullpath) - let err = loadRodFile(rod, g[m].fromDisk, conf, ignoreConfig = conf.cmd == cmdM) - if err == ok: - if conf.cmd == cmdM: - setupLookupTables(g, conf, cache, fileIdx, g[m]) - cachedModules.add fileIdx - g[m].status = loaded - result = false - else: - result = optForceFullMake in conf.globalOptions - # check its dependencies: - let imp = g[m].fromDisk.imports - for dep in imp: - let fid = toFileIndex(dep, g[m].fromDisk, conf) - # Warning: we need to traverse the full graph, so - # do **not use break here**! - if needsRecompile(g, conf, cache, fid, cachedModules): - result = true - - if not result: - setupLookupTables(g, conf, cache, fileIdx, g[m]) - cachedModules.add fileIdx - g[m].status = loaded - else: - g.pm[m] = LoadedModule(status: outdated, module: g[m].module) - else: - loadError(err, rod, conf) - g[m].status = outdated - result = true - when false: loadError(err, rod, conf) - of loading, loaded: - # For loading: Assume no recompile is required. - result = false - of outdated, storing, stored: - result = true - -proc moduleFromRodFile*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; - fileIdx: FileIndex; cachedModules: var seq[FileIndex]): PSym = - ## Returns 'nil' if the module needs to be recompiled. - bench g.depAnalysis: - if needsRecompile(g, conf, cache, fileIdx, cachedModules): - result = nil - else: - result = g[int fileIdx].module - assert result != nil - assert result.position == int(fileIdx) - for m in cachedModules: - loadToReplayNodes(g, conf, cache, m, g[int m]) - -template setupDecoder() {.dirty.} = - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - -proc loadProcBody*(config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; s: PSym): PNode = - bench g.loadBody: - let mId = s.itemId.module - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - let pos = g[mId].fromDisk.syms[s.itemId.item].ast - assert pos != emptyNodeId - result = loadProcBody(decoder, g, mId, g[mId].fromDisk.bodies, NodePos pos) - -proc loadTypeFromId*(config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: int; id: PackedItemId): PType = - bench g.loadType: - result = g[module].types.getOrDefault(id.item) - if result == nil: - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - result = loadType(decoder, g, module, id) - -proc loadSymFromId*(config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: int; id: PackedItemId): PSym = - bench g.loadSym: - result = g[module].syms.getOrDefault(id.item) - if result == nil: - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - result = loadSym(decoder, g, module, id) - -proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; config: ConfigRef): ItemId = - if id.module == LitId(0): - ItemId(module: thisModule.int32, item: id.item) - else: - ItemId(module: toFileIndex(id.module, g[thisModule].fromDisk, config).int32, item: id.item) - -proc simulateLoadedModule*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; - moduleSym: PSym; m: PackedModule) = - # For now only used for heavy debugging. In the future we could use this to reduce the - # compiler's memory consumption. - let idx = moduleSym.position - assert g[idx].status in {storing} - g[idx].status = loaded - assert g[idx].module == moduleSym - setupLookupTables(g, conf, cache, FileIndex(idx), g[idx]) - loadToReplayNodes(g, conf, cache, FileIndex(idx), g[idx]) - -# ---------------- symbol table handling ---------------- - -type - RodIter* = object - decoder: PackedDecoder - values: seq[PackedItemId] - i, module: int - -template interfSelect(a: LoadedModule, importHidden: bool): auto = - var ret = a.iface.addr - if importHidden: ret = a.ifaceHidden.addr - ret[] - -proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: FileIndex; - name: PIdent, importHidden: bool): PSym = - it.decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - it.values = g[int module].interfSelect(importHidden).getOrDefault(name) - it.i = 0 - it.module = int(module) - if it.i < it.values.len: - result = loadSym(it.decoder, g, int(module), it.values[it.i]) - inc it.i - else: - result = nil - -proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: FileIndex; importHidden: bool): PSym = - it.decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - it.values = @[] - it.module = int(module) - for v in g[int module].interfSelect(importHidden).values: - it.values.add v - it.i = 0 - if it.i < it.values.len: - result = loadSym(it.decoder, g, int(module), it.values[it.i]) - inc it.i - else: - result = nil - -proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym = - if it.i < it.values.len: - result = loadSym(it.decoder, g, it.module, it.values[it.i]) - inc it.i - else: - result = nil - -iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: FileIndex; - name: PIdent, importHidden: bool): PSym = - setupDecoder() - let values = g[int module].interfSelect(importHidden).getOrDefault(name) - for pid in values: - let s = loadSym(decoder, g, int(module), pid) - assert s != nil - yield s - -proc interfaceSymbol*(config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: FileIndex; - name: PIdent, importHidden: bool): PSym = - setupDecoder() - let values = g[int module].interfSelect(importHidden).getOrDefault(name) - result = loadSym(decoder, g, int(module), values[0]) - -proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator = - IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.syms.len, - typeId: int32 m.fromDisk.types.len) - -proc searchForCompilerproc*(m: LoadedModule; name: string): int32 = - # slow, linear search, but the results are cached: - for it in items(m.fromDisk.compilerProcs): - if m.fromDisk.strings[it[0]] == name: - return it[1] - return -1 - -# ------------------------- .rod file viewer --------------------------------- - -proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = - var m: PackedModule = PackedModule() - let err = loadRodFile(rodfile, m, config, ignoreConfig=true) - if err != ok: - config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err - - when false: - echo "exports:" - for ex in m.exports: - echo " ", m.strings[ex[0]], " local ID: ", ex[1] - assert ex[0] == m.syms[ex[1]].name - # ex[1] int32 - - echo "reexports:" - for ex in m.reexports: - echo " ", m.strings[ex[0]] - # reexports*: seq[(LitId, PackedItemId)] - - echo "hidden: " & $m.hidden.len - for ex in m.hidden: - echo " ", m.strings[ex[0]], " local ID: ", ex[1] - - when false: - echo "all symbols" - for i in 0..high(m.syms): - if m.syms[i].name != LitId(0): - echo " ", m.strings[m.syms[i].name], " local ID: ", i, " kind ", m.syms[i].kind - else: - echo " local ID: ", i, " kind ", m.syms[i].kind - - echo "symbols: ", m.syms.len, " types: ", m.types.len, - " top level nodes: ", m.topLevel.len, " other nodes: ", m.bodies.len, - " strings: ", m.strings.len, " numbers: ", m.numbers.len - - echo "SIZES:" - echo "symbols: ", m.syms.len * sizeof(PackedSym), " types: ", m.types.len * sizeof(PackedType), - " top level nodes: ", m.topLevel.len * sizeof(PackedNode), - " other nodes: ", m.bodies.len * sizeof(PackedNode), - " strings: ", sizeOnDisc(m.strings) - when false: - var tt = 0 - var fc = 0 - for x in m.topLevel: - if x.kind == nkSym or x.typeId == nilItemId: inc tt - if x.flags == {}: inc fc - for x in m.bodies: - if x.kind == nkSym or x.typeId == nilItemId: inc tt - if x.flags == {}: inc fc - let total = float(m.topLevel.len + m.bodies.len) - echo "nodes with nil type: ", tt, " in % ", tt.float / total - echo "nodes with empty flags: ", fc.float / total diff --git a/compiler/ic/iclineinfos.nim b/compiler/ic/iclineinfos.nim deleted file mode 100644 index 74a7d971be2c..000000000000 --- a/compiler/ic/iclineinfos.nim +++ /dev/null @@ -1,84 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2024 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# For the line information we use 32 bits. They are used as follows: -# Bit 0 (AsideBit): If we have inline line information or not. If not, the -# remaining 31 bits are used as an index into a seq[(LitId, int, int)]. -# -# We use 10 bits for the "file ID", this means a program can consist of as much -# as 1024 different files. (If it uses more files than that, the overflow bit -# would be set.) -# This means we have 21 bits left to encode the (line, col) pair. We use 7 bits for the column -# so 128 is the limit and 14 bits for the line number. -# The packed representation supports files with up to 16384 lines. -# Keep in mind that whenever any limit is reached the AsideBit is set and the real line -# information is kept in a side channel. - -import std / assertions - -const - AsideBit = 1 - FileBits = 10 - LineBits = 14 - ColBits = 7 - FileMax = (1 shl FileBits) - 1 - LineMax = (1 shl LineBits) - 1 - ColMax = (1 shl ColBits) - 1 - -static: - assert AsideBit + FileBits + LineBits + ColBits == 32 - -import .. / ic / [bitabs, rodfiles] # for LitId - -type - PackedLineInfo* = distinct uint32 - - LineInfoManager* = object - aside: seq[(LitId, int32, int32)] - -const - NoLineInfo* = PackedLineInfo(0'u32) - -proc pack*(m: var LineInfoManager; file: LitId; line, col: int32): PackedLineInfo = - if file.uint32 <= FileMax.uint32 and line <= LineMax and col <= ColMax: - let col = if col < 0'i32: 0'u32 else: col.uint32 - let line = if line < 0'i32: 0'u32 else: line.uint32 - # use inline representation: - result = PackedLineInfo((file.uint32 shl 1'u32) or (line shl uint32(AsideBit + FileBits)) or - (col shl uint32(AsideBit + FileBits + LineBits))) - else: - result = PackedLineInfo((m.aside.len shl 1) or AsideBit) - m.aside.add (file, line, col) - -proc unpack*(m: LineInfoManager; i: PackedLineInfo): (LitId, int32, int32) = - let i = i.uint32 - if (i and 1'u32) == 0'u32: - # inline representation: - result = (LitId((i shr 1'u32) and FileMax.uint32), - int32((i shr uint32(AsideBit + FileBits)) and LineMax.uint32), - int32((i shr uint32(AsideBit + FileBits + LineBits)) and ColMax.uint32)) - else: - result = m.aside[int(i shr 1'u32)] - -proc getFileId*(m: LineInfoManager; i: PackedLineInfo): LitId = - result = unpack(m, i)[0] - -proc store*(r: var RodFile; m: LineInfoManager) = storeSeq(r, m.aside) -proc load*(r: var RodFile; m: var LineInfoManager) = loadSeq(r, m.aside) - -when isMainModule: - var m = LineInfoManager(aside: @[]) - for i in 0'i32..<16388'i32: - for col in 0'i32..<100'i32: - let packed = pack(m, LitId(1023), i, col) - let u = unpack(m, packed) - assert u[0] == LitId(1023) - assert u[1] == i - assert u[2] == col - echo m.aside.len diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim deleted file mode 100644 index 3e8ea25034db..000000000000 --- a/compiler/ic/integrity.nim +++ /dev/null @@ -1,155 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2021 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Integrity checking for a set of .rod files. -## The set must cover a complete Nim project. - -import std/[sets, tables] - -when defined(nimPreviewSlimSystem): - import std/assertions - -import ".." / [ast, modulegraphs] -import packed_ast, bitabs, ic - -type - CheckedContext = object - g: ModuleGraph - thisModule: int32 - checkedSyms: HashSet[ItemId] - checkedTypes: HashSet[ItemId] - -proc checkType(c: var CheckedContext; typeId: PackedItemId) -proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) -proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) - -proc checkTypeObj(c: var CheckedContext; typ: PackedType) = - for child in typ.types: - checkType(c, child) - if typ.n != emptyNodeId: - checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos typ.n) - if typ.sym != nilItemId: - checkForeignSym(c, typ.sym) - if typ.owner != nilItemId: - checkForeignSym(c, typ.owner) - checkType(c, typ.typeInst) - -proc checkType(c: var CheckedContext; typeId: PackedItemId) = - if typeId == nilItemId: return - let itemId = translateId(typeId, c.g.packed, c.thisModule, c.g.config) - if not c.checkedTypes.containsOrIncl(itemId): - let oldThisModule = c.thisModule - c.thisModule = itemId.module - checkTypeObj c, c.g.packed[itemId.module].fromDisk.types[itemId.item] - c.thisModule = oldThisModule - -proc checkSym(c: var CheckedContext; s: PackedSym) = - if s.name != LitId(0): - assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId s.name - checkType c, s.typ - if s.ast != emptyNodeId: - checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos s.ast) - if s.owner != nilItemId: - checkForeignSym(c, s.owner) - -proc checkLocalSym(c: var CheckedContext; item: int32) = - let itemId = ItemId(module: c.thisModule, item: item) - if not c.checkedSyms.containsOrIncl(itemId): - checkSym c, c.g.packed[c.thisModule].fromDisk.syms[item] - -proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) = - let itemId = translateId(symId, c.g.packed, c.thisModule, c.g.config) - if not c.checkedSyms.containsOrIncl(itemId): - let oldThisModule = c.thisModule - c.thisModule = itemId.module - checkSym c, c.g.packed[itemId.module].fromDisk.syms[itemId.item] - c.thisModule = oldThisModule - -proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) = - let t = findType(tree, n) - if t != nilItemId: - checkType(c, t) - case n.kind - of nkEmpty, nkNilLit, nkType, nkNilRodNode: - discard - of nkIdent: - assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId - of nkSym: - checkLocalSym(c, tree[n].soperand) - of directIntLit: - discard - of externIntLit, nkFloatLit..nkFloat128Lit: - assert c.g.packed[c.thisModule].fromDisk.numbers.hasLitId n.litId - of nkStrLit..nkTripleStrLit: - assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId - of nkModuleRef: - let (n1, n2) = sons2(tree, n) - assert n1.kind == nkNone - assert n2.kind == nkNone - checkForeignSym(c, PackedItemId(module: n1.litId, item: tree[n2].soperand)) - else: - for n0 in sonsReadonly(tree, n): - checkNode(c, tree, n0) - -proc checkTree(c: var CheckedContext; t: PackedTree) = - for p in allNodes(t): checkNode(c, t, p) - -proc checkLocalSymIds(c: var CheckedContext; m: PackedModule; symIds: seq[int32]) = - for symId in symIds: - assert symId >= 0 and symId < m.syms.len, $symId & " " & $m.syms.len - -proc checkModule(c: var CheckedContext; m: PackedModule) = - # We check that: - # - Every symbol references existing types and symbols. - # - Every tree node references existing types and symbols. - for _, v in pairs(m.syms): - checkLocalSym c, v.id - - checkTree c, m.toReplay - checkTree c, m.topLevel - - for e in m.exports: - #assert e[1] >= 0 and e[1] < m.syms.len - assert e[0] == m.syms[e[1]].name - - for e in m.compilerProcs: - #assert e[1] >= 0 and e[1] < m.syms.len - assert e[0] == m.syms[e[1]].name - - checkLocalSymIds c, m, m.converters - checkLocalSymIds c, m, m.methods - checkLocalSymIds c, m, m.trmacros - checkLocalSymIds c, m, m.pureEnums - #[ - To do: Check all these fields: - - reexports*: seq[(LitId, PackedItemId)] - macroUsages*: seq[(PackedItemId, PackedLineInfo)] - - typeInstCache*: seq[(PackedItemId, PackedItemId)] - procInstCache*: seq[PackedInstantiation] - attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)] - methodsPerGenericType*: seq[(PackedItemId, int, PackedItemId)] - enumToStringProcs*: seq[(PackedItemId, PackedItemId)] - methodsPerType*: seq[(PackedItemId, PackedItemId)] - dispatchers*: seq[PackedItemId] - ]# - -proc checkIntegrity*(g: ModuleGraph) = - var c = CheckedContext(g: g) - for i in 0..= currentCol and col < currentCol+tokenLen: - result = true - else: - result = false - else: - result = false - -proc searchLocalSym(c: var NavContext; s: PackedSym; info: PackedLineInfo): bool = - result = s.name != LitId(0) and - isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[c.thisModule].fromDisk.strings[s.name].len) - -proc searchForeignSym(c: var NavContext; s: ItemId; info: PackedLineInfo): bool = - let name = c.g.packed[s.module].fromDisk.syms[s.item].name - result = name != LitId(0) and - isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[s.module].fromDisk.strings[name].len) - -const - EmptyItemId = ItemId(module: -1'i32, item: -1'i32) - -proc search(c: var NavContext; tree: PackedTree): ItemId = - # We use the linear representation here directly: - for i in 0..= 0 and tree[n].kind in declarativeNodes - -proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) = - let (fileId, line, col) = unpack(c.g.packed[c.thisModule].fromDisk.man, info) - var m = "" - var file = c.g.packed[c.thisModule].fromDisk.strings[fileId] - if c.outputSep == ' ': - file = os.extractFilename file - toLocation(m, file, line, col + ColOffset) - if not c.alreadyEmitted.containsOrIncl(m): - msgWriteln c.g.config, (if isDecl: "def" else: "usage") & c.outputSep & m - -proc list(c: var NavContext; tree: PackedTree; sym: ItemId) = - for i in 0.. nkNilLit - let distance = int32(tree.nodes.len - pos) - assert distance > 0 - tree.nodes[pos].x = toX(k, cast[uint32](distance)) - -proc len*(tree: PackedTree): int {.inline.} = tree.nodes.len - -proc `[]`*(tree: PackedTree; i: NodePos): lent PackedNode {.inline.} = - tree.nodes[i.int] - -template rawSpan(n: PackedNode): int = int(uoperand(n)) - -proc nextChild(tree: PackedTree; pos: var int) {.inline.} = - if tree.nodes[pos].kind > nkNilLit: - assert tree.nodes[pos].uoperand > 0 - inc pos, tree.nodes[pos].rawSpan - else: - inc pos - -iterator sonsReadonly*(tree: PackedTree; n: NodePos): NodePos = - var pos = n.int - assert tree.nodes[pos].kind > nkNilLit - let last = pos + tree.nodes[pos].rawSpan - inc pos - while pos < last: - yield NodePos pos - nextChild tree, pos - -iterator sons*(dest: var PackedTree; tree: PackedTree; n: NodePos): NodePos = - let patchPos = prepare(dest, tree, n) - for x in sonsReadonly(tree, n): yield x - patch dest, patchPos - -iterator isons*(dest: var PackedTree; tree: PackedTree; - n: NodePos): (int, NodePos) = - var i = 0 - for ch0 in sons(dest, tree, n): - yield (i, ch0) - inc i - -iterator sonsFrom1*(tree: PackedTree; n: NodePos): NodePos = - var pos = n.int - assert tree.nodes[pos].kind > nkNilLit - let last = pos + tree.nodes[pos].rawSpan - inc pos - if pos < last: - nextChild tree, pos - while pos < last: - yield NodePos pos - nextChild tree, pos - -iterator sonsWithoutLast2*(tree: PackedTree; n: NodePos): NodePos = - var count = 0 - for child in sonsReadonly(tree, n): - inc count - var pos = n.int - assert tree.nodes[pos].kind > nkNilLit - let last = pos + tree.nodes[pos].rawSpan - inc pos - while pos < last and count > 2: - yield NodePos pos - dec count - nextChild tree, pos - -proc parentImpl(tree: PackedTree; n: NodePos): NodePos = - # finding the parent of a node is rather easy: - var pos = n.int - 1 - while pos >= 0 and (isAtom(tree, pos) or (pos + tree.nodes[pos].rawSpan - 1 < n.int)): - dec pos - #assert pos >= 0, "node has no parent" - result = NodePos(pos) - -template parent*(n: NodePos): NodePos = parentImpl(tree, n) - -proc hasXsons*(tree: PackedTree; n: NodePos; x: int): bool = - var count = 0 - if tree.nodes[n.int].kind > nkNilLit: - for child in sonsReadonly(tree, n): inc count - result = count == x - -proc hasAtLeastXsons*(tree: PackedTree; n: NodePos; x: int): bool = - if tree.nodes[n.int].kind > nkNilLit: - var count = 0 - for child in sonsReadonly(tree, n): - inc count - if count >= x: return true - return false - -proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} = - NodePos(n.int+1) -proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} = - tree.nodes[n.int].kind -proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} = - LitId tree.nodes[n.int].uoperand -proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} = - tree.nodes[n.int].info - -proc findType*(tree: PackedTree; n: NodePos): PackedItemId = - for x in tree.withTypes: - if x[0] == int32(n): return x[1] - if x[0] > int32(n): return nilItemId - return nilItemId - -proc findFlags*(tree: PackedTree; n: NodePos): TNodeFlags = - for x in tree.withFlags: - if x[0] == int32(n): return x[1] - if x[0] > int32(n): return {} - return {} - -template typ*(n: NodePos): PackedItemId = - tree.findType(n) -template flags*(n: NodePos): TNodeFlags = - tree.findFlags(n) - -template uoperand*(n: NodePos): uint32 = - tree.nodes[n.int].uoperand - -proc span*(tree: PackedTree; pos: int): int {.inline.} = - if isAtom(tree, pos): 1 else: tree.nodes[pos].rawSpan - -proc sons2*(tree: PackedTree; n: NodePos): (NodePos, NodePos) = - assert(not isAtom(tree, n.int)) - let a = n.int+1 - let b = a + span(tree, a) - result = (NodePos a, NodePos b) - -proc sons3*(tree: PackedTree; n: NodePos): (NodePos, NodePos, NodePos) = - assert(not isAtom(tree, n.int)) - let a = n.int+1 - let b = a + span(tree, a) - let c = b + span(tree, b) - result = (NodePos a, NodePos b, NodePos c) - -proc ithSon*(tree: PackedTree; n: NodePos; i: int): NodePos = - result = default(NodePos) - if tree.nodes[n.int].kind > nkNilLit: - var count = 0 - for child in sonsReadonly(tree, n): - if count == i: return child - inc count - assert false, "node has no i-th child" - -when false: - proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} = - tree.sh.strings[lit] - -template kind*(n: NodePos): TNodeKind = tree.nodes[n.int].kind -template info*(n: NodePos): PackedLineInfo = tree.nodes[n.int].info -template litId*(n: NodePos): LitId = LitId tree.nodes[n.int].uoperand - -template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].soperand - -proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1) - -const - externIntLit* = {nkCharLit, - nkIntLit, - nkInt8Lit, - nkInt16Lit, - nkInt32Lit, - nkInt64Lit, - nkUIntLit, - nkUInt8Lit, - nkUInt16Lit, - nkUInt32Lit, - nkUInt64Lit} - - externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt32Lit, nkInt64Lit} - externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit} - directIntLit* = nkNone - -template copyInto*(dest, n, body) = - let patchPos = prepare(dest, tree, n) - body - patch dest, patchPos - -template copyIntoKind*(dest, kind, info, body) = - let patchPos = prepare(dest, kind, info) - body - patch dest, patchPos - -proc getNodeId*(tree: PackedTree): NodeId {.inline.} = NodeId tree.nodes.len - -iterator allNodes*(tree: PackedTree): NodePos = - var p = 0 - while p < tree.len: - yield NodePos(p) - let s = span(tree, p) - inc p, s - -proc toPackedItemId*(item: int32): PackedItemId {.inline.} = - PackedItemId(module: LitId(0), item: item) - -proc load*(f: var RodFile; t: var PackedTree) = - loadSeq f, t.nodes - loadSeq f, t.withFlags - loadSeq f, t.withTypes - -proc store*(f: var RodFile; t: PackedTree) = - storeSeq f, t.nodes - storeSeq f, t.withFlags - storeSeq f, t.withTypes diff --git a/compiler/ic/replayer.nim b/compiler/ic/replayer.nim deleted file mode 100644 index b244ec885c0d..000000000000 --- a/compiler/ic/replayer.nim +++ /dev/null @@ -1,171 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2020 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Module that contains code to replay global VM state changes and pragma -## state like ``{.compile: "foo.c".}``. For IC (= Incremental compilation) -## support. - -import ".." / [ast, modulegraphs, trees, extccomp, btrees, - msgs, lineinfos, pathutils, options, cgmeth] - -import std/tables - -when defined(nimPreviewSlimSystem): - import std/assertions - -import packed_ast, ic, bitabs - -proc replayStateChanges*(module: PSym; g: ModuleGraph) = - let list = module.ast - assert list != nil - assert list.kind == nkStmtList - for n in list: - assert n.kind == nkReplayAction - # Fortunately only a tiny subset of the available pragmas need to - # be replayed here. This is always a subset of ``pragmas.stmtPragmas``. - if n.len >= 2: - internalAssert g.config, n[0].kind == nkStrLit and n[1].kind == nkStrLit - case n[0].strVal - of "hint": message(g.config, n.info, hintUser, n[1].strVal) - of "warning": message(g.config, n.info, warnUser, n[1].strVal) - of "error": localError(g.config, n.info, errUser, n[1].strVal) - of "compile": - internalAssert g.config, n.len == 4 and n[2].kind == nkStrLit - let cname = AbsoluteFile n[1].strVal - var cf = Cfile(nimname: splitFile(cname).name, cname: cname, - obj: AbsoluteFile n[2].strVal, - flags: {CfileFlag.External}, - customArgs: n[3].strVal) - extccomp.addExternalFileToCompile(g.config, cf) - of "link": - extccomp.addExternalFileToLink(g.config, AbsoluteFile n[1].strVal) - of "passl": - extccomp.addLinkOption(g.config, n[1].strVal) - of "passc": - extccomp.addCompileOption(g.config, n[1].strVal) - of "localpassc": - extccomp.addLocalCompileOption(g.config, n[1].strVal, toFullPathConsiderDirty(g.config, module.info.fileIndex)) - of "cppdefine": - options.cppDefine(g.config, n[1].strVal) - of "inc": - let destKey = n[1].strVal - let by = n[2].intVal - let v = getOrDefault(g.cacheCounters, destKey) - g.cacheCounters[destKey] = v+by - of "put": - let destKey = n[1].strVal - let key = n[2].strVal - let val = n[3] - if not contains(g.cacheTables, destKey): - g.cacheTables[destKey] = initBTree[string, PNode]() - if not contains(g.cacheTables[destKey], key): - g.cacheTables[destKey].add(key, val) - else: - internalError(g.config, n.info, "key already exists: " & key) - of "incl": - let destKey = n[1].strVal - let val = n[2] - if not contains(g.cacheSeqs, destKey): - g.cacheSeqs[destKey] = newTree(nkStmtList, val) - else: - block search: - for existing in g.cacheSeqs[destKey]: - if exprStructuralEquivalent(existing, val, strictSymEquality=true): - break search - g.cacheSeqs[destKey].add val - of "add": - let destKey = n[1].strVal - let val = n[2] - if not contains(g.cacheSeqs, destKey): - g.cacheSeqs[destKey] = newTree(nkStmtList, val) - else: - g.cacheSeqs[destKey].add val - else: - internalAssert g.config, false - -proc replayBackendProcs*(g: ModuleGraph; module: int) = - for it in mitems(g.packed[module].fromDisk.attachedOps): - let key = translateId(it[0], g.packed, module, g.config) - let op = it[1] - let tmp = translateId(it[2], g.packed, module, g.config) - let symId = FullId(module: tmp.module, packed: it[2]) - g.attachedOps[op][key] = LazySym(id: symId, sym: nil) - - for it in mitems(g.packed[module].fromDisk.enumToStringProcs): - let key = translateId(it[0], g.packed, module, g.config) - let tmp = translateId(it[1], g.packed, module, g.config) - let symId = FullId(module: tmp.module, packed: it[1]) - g.enumToStringProcs[key] = LazySym(id: symId, sym: nil) - - for it in mitems(g.packed[module].fromDisk.methodsPerType): - let key = translateId(it[0], g.packed, module, g.config) - let tmp = translateId(it[1], g.packed, module, g.config) - let symId = FullId(module: tmp.module, packed: it[1]) - g.methodsPerType.mgetOrPut(key, @[]).add LazySym(id: symId, sym: nil) - - for it in mitems(g.packed[module].fromDisk.dispatchers): - let tmp = translateId(it, g.packed, module, g.config) - let symId = FullId(module: tmp.module, packed: it) - g.dispatchers.add LazySym(id: symId, sym: nil) - -proc replayGenericCacheInformation*(g: ModuleGraph; module: int) = - ## We remember the generic instantiations a module performed - ## in order to to avoid the code bloat that generic code tends - ## to imply. This is cheaper than deduplication of identical - ## generic instantiations. However, deduplication is more - ## powerful and general and I hope to implement it soon too - ## (famous last words). - assert g.packed[module].status == loaded - for it in g.packed[module].fromDisk.typeInstCache: - let key = translateId(it[0], g.packed, module, g.config) - g.typeInstCache.mgetOrPut(key, @[]).add LazyType(id: FullId(module: module, packed: it[1]), typ: nil) - - for it in mitems(g.packed[module].fromDisk.procInstCache): - let key = translateId(it.key, g.packed, module, g.config) - let sym = translateId(it.sym, g.packed, module, g.config) - var concreteTypes = newSeq[FullId](it.concreteTypes.len) - for i in 0..high(it.concreteTypes): - let tmp = translateId(it.concreteTypes[i], g.packed, module, g.config) - concreteTypes[i] = FullId(module: tmp.module, packed: it.concreteTypes[i]) - - g.procInstCache.mgetOrPut(key, @[]).add LazyInstantiation( - module: module, sym: FullId(module: sym.module, packed: it.sym), - concreteTypes: concreteTypes, inst: nil) - - for it in mitems(g.packed[module].fromDisk.methodsPerGenericType): - let key = translateId(it[0], g.packed, module, g.config) - let col = it[1] - let tmp = translateId(it[2], g.packed, module, g.config) - let symId = FullId(module: tmp.module, packed: it[2]) - g.methodsPerGenericType.mgetOrPut(key, @[]).add (col, LazySym(id: symId, sym: nil)) - - replayBackendProcs(g, module) - - for it in mitems(g.packed[module].fromDisk.methods): - let sym = loadSymFromId(g.config, g.cache, g.packed, module, - PackedItemId(module: LitId(0), item: it)) - methodDef(g, g.idgen, sym) - - when false: - # not used anymore: - for it in mitems(g.packed[module].fromDisk.compilerProcs): - let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it[1])) - g.lazyCompilerprocs[g.packed[module].fromDisk.sh.strings[it[0]]] = symId - - for it in mitems(g.packed[module].fromDisk.converters): - let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it)) - g.ifaces[module].converters.add LazySym(id: symId, sym: nil) - - for it in mitems(g.packed[module].fromDisk.trmacros): - let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it)) - g.ifaces[module].patterns.add LazySym(id: symId, sym: nil) - - for it in mitems(g.packed[module].fromDisk.pureEnums): - let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it)) - g.ifaces[module].pureEnums.add LazySym(id: symId, sym: nil) diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim deleted file mode 100644 index ac995dd2eb4a..000000000000 --- a/compiler/ic/rodfiles.nim +++ /dev/null @@ -1,283 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2020 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Low level binary format used by the compiler to store and load various AST -## and related data. -## -## NB: this is incredibly low level and if you're interested in how the -## compiler works and less a storage format, you're probably looking for -## the `ic` or `packed_ast` modules to understand the logical format. - -from std/typetraits import supportsCopyMem - -when defined(nimPreviewSlimSystem): - import std/[syncio, assertions] - -import std / tables - -## Overview -## ======== -## `RodFile` represents a Rod File (versioned binary format), and the -## associated data for common interactions such as IO and error tracking -## (`RodFileError`). The file format broken up into sections (`RodSection`) -## and preceded by a header (see: `cookie`). The precise layout, section -## ordering and data following the section are determined by the user. See -## `ic.loadRodFile`. -## -## A basic but "wrong" example of the lifecycle: -## --------------------------------------------- -## 1. `create` or `open` - create a new one or open an existing -## 2. `storeHeader` - header info -## 3. `storePrim` or `storeSeq` - save your stuff -## 4. `close` - and we're done -## -## Now read the bits below to understand what's missing. -## -## ### Issues with the Example -## Missing Sections: -## This is a low level API, so headers and sections need to be stored and -## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` & -## `loadSection`, respectively. -## -## No Error Handling: -## The API is centered around IO and prone to error, each operation checks or -## sets the `RodFile.err` field. A user of this API needs to handle these -## appropriately. -## -## API Notes -## ========= -## -## Valid inputs for Rod files -## -------------------------- -## ASTs, hopes, dreams, and anything as long as it and any children it may have -## support `copyMem`. This means anything that is not a pointer and that does not contain a pointer. At a glance these are: -## * string -## * objects & tuples (fields are recursed) -## * sequences AKA `seq[T]` -## -## Note on error handling style -## ---------------------------- -## A flag based approach is used where operations no-op in case of a -## preexisting error and set the flag if they encounter one. -## -## Misc -## ---- -## * 'Prim' is short for 'primitive', as in a non-sequence type - -type - RodSection* = enum - versionSection - configSection - stringsSection - checkSumsSection - depsSection - numbersSection - exportsSection - hiddenSection - reexportsSection - compilerProcsSection - trmacrosSection - convertersSection - methodsSection - pureEnumsSection - toReplaySection - topLevelSection - bodiesSection - symsSection - typesSection - typeInstCacheSection - procInstCacheSection - attachedOpsSection - methodsPerGenericTypeSection - enumToStringProcsSection - methodsPerTypeSection - dispatchersSection - typeInfoSection # required by the backend - backendFlagsSection - aliveSymsSection # beware, this is stored in a `.alivesyms` file. - sideChannelSection - namespaceSection - symnamesSection - - RodFileError* = enum - ok, tooBig, cannotOpen, ioFailure, wrongHeader, wrongSection, configMismatch, - includeFileChanged - - RodFile* = object - f*: File - currentSection*: RodSection # for error checking - err*: RodFileError # little experiment to see if this works - # better than exceptions. - -const - RodVersion = 2 - defaultCookie = [byte(0), byte('R'), byte('O'), byte('D'), - byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)] - -proc setError(f: var RodFile; err: RodFileError) {.inline.} = - f.err = err - #raise newException(IOError, "IO error") - -proc storePrim*(f: var RodFile; s: string) = - ## Stores a string. - ## The len is prefixed to allow for later retreival. - if f.err != ok: return - if s.len >= high(int32): - setError f, tooBig - return - var lenPrefix = int32(s.len) - if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - setError f, ioFailure - else: - if s.len != 0: - if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len: - setError f, ioFailure - -proc storePrim*[T](f: var RodFile; x: T) = - ## Stores a non-sequence/string `T`. - ## If `T` doesn't support `copyMem` and is an object or tuple then the fields - ## are written -- the user from context will need to know which `T` to load. - if f.err != ok: return - when supportsCopyMem(T): - if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): - setError f, ioFailure - elif T is tuple: - for y in fields(x): - storePrim(f, y) - elif T is object: - for y in fields(x): - when y is seq: - storeSeq(f, y) - else: - storePrim(f, y) - else: - {.error: "unsupported type for 'storePrim'".} - -proc storeSeq*[T](f: var RodFile; s: seq[T]) = - ## Stores a sequence of `T`s, with the len as a prefix for later retrieval. - if f.err != ok: return - if s.len >= high(int32): - setError f, tooBig - return - var lenPrefix = int32(s.len) - if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - setError f, ioFailure - else: - for i in 0..= high(int32): - setError f, tooBig - return - var lenPrefix = int32(s.len) - if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - setError f, ioFailure - else: - for _, v in s: - storePrim(f, v) - -proc loadPrim*(f: var RodFile; s: var string) = - ## Read a string, the length was stored as a prefix - if f.err != ok: return - var lenPrefix = int32(0) - if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - setError f, ioFailure - else: - s = newString(lenPrefix) - if lenPrefix > 0: - if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len: - setError f, ioFailure - -proc loadPrim*[T](f: var RodFile; x: var T) = - ## Load a non-sequence/string `T`. - if f.err != ok: return - when supportsCopyMem(T): - if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): - setError f, ioFailure - elif T is tuple: - for y in fields(x): - loadPrim(f, y) - elif T is object: - for y in fields(x): - when y is seq: - loadSeq(f, y) - else: - loadPrim(f, y) - else: - {.error: "unsupported type for 'loadPrim'".} - -proc loadSeq*[T](f: var RodFile; s: var seq[T]) = - ## `T` must be compatible with `copyMem`, see `loadPrim` - if f.err != ok: return - var lenPrefix = int32(0) - if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - setError f, ioFailure - else: - s = newSeq[T](lenPrefix) - for i in 0..= g.ifaces.len: setLen(g.ifaces, m.position + 1) - if m.position >= g.packed.len: - setLen(g.packed.pm, m.position + 1) - g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[], uniqueName: rope(uniqueModuleName(g.config, m))) initStrTables(g, m) -proc registerModuleById*(g: ModuleGraph; m: FileIndex) = - registerModule(g, g.packed[int m].module) - proc initOperators*(g: ModuleGraph): Operators = # These are safe for IC. # Public because it's used by DrNim. @@ -601,7 +543,6 @@ proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool = proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} = result = s.ast[bodyPos] - assert result != nil proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, fromModule: PSym, ) = let conf = graph.config diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 87adaac39126..cb80bda6aa0d 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -332,21 +332,21 @@ proc inclSym(sq: var seq[PSym], s: PSym): bool = sq.add s result = true -proc addConverter*(c: PContext, conv: LazySym) = - assert conv.sym != nil - if inclSym(c.converters, conv.sym): +proc addConverter*(c: PContext, conv: PSym) = + assert conv != nil + if inclSym(c.converters, conv): add(c.graph.ifaces[c.module.position].converters, conv) -proc addConverterDef*(c: PContext, conv: LazySym) = +proc addConverterDef*(c: PContext, conv: PSym) = addConverter(c, conv) -proc addPureEnum*(c: PContext, e: LazySym) = - assert e.sym != nil +proc addPureEnum*(c: PContext, e: PSym) = + assert e != nil add(c.graph.ifaces[c.module.position].pureEnums, e) -proc addPattern*(c: PContext, p: LazySym) = - assert p.sym != nil - if inclSym(c.patterns, p.sym): +proc addPattern*(c: PContext, p: PSym) = + assert p != nil + if inclSym(c.patterns, p): add(c.graph.ifaces[c.module.position].patterns, p) proc exportSym*(c: PContext; s: PSym) = @@ -570,7 +570,7 @@ template addExport*(c: PContext; s: PSym) = addExport(c.graph, c.module, s) proc addToGenericProcCache*(c: PContext; s: PSym; inst: PInstantiation) = - c.graph.procInstCache.mgetOrPut(s.itemId, @[]).add LazyInstantiation(module: c.module.position, inst: inst) + c.graph.procInstCache.mgetOrPut(s.itemId, @[]).add inst proc addToGenericCache*(c: PContext; s: PSym; inst: PType) = - c.graph.typeInstCache.mgetOrPut(s.itemId, @[]).add LazyType(typ: inst) \ No newline at end of file + c.graph.typeInstCache.mgetOrPut(s.itemId, @[]).add inst \ No newline at end of file diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index ea32dae0e4f3..68e9c35a5daa 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2986,9 +2986,9 @@ proc semExportExcept(c: PContext, n: PNode): PNode = proc semExport(c: PContext, n: PNode): PNode = proc specialSyms(c: PContext; s: PSym) {.inline.} = - if s.kind == skConverter: addConverter(c, LazySym(sym: s)) + if s.kind == skConverter: addConverter(c, s) elif s.kind == skType and s.typ != nil and s.typ.kind == tyEnum and sfPure in s.flags: - addPureEnum(c, LazySym(sym: s)) + addPureEnum(c, s) result = newNodeI(nkExportStmt, n.info) for i in 0..= y.depth: 1 @@ -105,7 +105,7 @@ proc collectVTableDispatchers*(g: ModuleGraph) = for item in g.objectTree[baseType.itemId]: if item.value.itemId notin itemTable: - itemTable[item.value.itemId] = newSeq[LazySym](methodIndexLen) + itemTable[item.value.itemId] = newSeq[PSym](methodIndexLen) var mIndex = 0 # here is the correpsonding index if baseType.itemId notin rootItemIdCount: @@ -115,13 +115,13 @@ proc collectVTableDispatchers*(g: ModuleGraph) = rootItemIdCount.inc(baseType.itemId) for idx in 0..= y.depth: 1 @@ -143,7 +143,7 @@ proc sortVTableDispatchers*(g: ModuleGraph) = for item in g.objectTree[baseType.itemId]: if item.value.itemId notin itemTable: - itemTable[item.value.itemId] = newSeq[LazySym](methodIndexLen) + itemTable[item.value.itemId] = newSeq[PSym](methodIndexLen) var mIndex = 0 # here is the correpsonding index if baseType.itemId notin rootItemIdCount: @@ -153,7 +153,7 @@ proc sortVTableDispatchers*(g: ModuleGraph) = rootItemIdCount.inc(baseType.itemId) for idx in 0.. Date: Wed, 30 Oct 2024 22:50:24 +0800 Subject: [PATCH 4/9] clean up --- compiler/ast.nim | 2 +- compiler/cgen.nim | 9 ++------- compiler/options.nim | 1 - 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 5ff986776c3e..00779a30965f 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -834,7 +834,7 @@ var gconfig {.threadvar.}: Gconfig proc setUseIc*(useIc: bool) = gconfig.useIc = useIc proc comment*(n: PNode): string = - if nfHasComment in n.flags and not gconfig.useIc: + if nfHasComment in n.flags: # IC doesn't track comments, see `packed_ast`, so this could fail result = gconfig.comments[n.nodeId] else: diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 0a2ec10a139d..7c99664a5b9a 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -61,13 +61,8 @@ proc addForwardedProc(m: BModule, prc: PSym) = m.g.forwardedProcs.add(prc) proc findPendingModule(m: BModule, s: PSym): BModule = - # TODO fixme - if m.config.symbolFiles == v2Sf: - let ms = s.itemId.module #getModule(s) - result = m.g.modules[ms] - else: - var ms = getModule(s) - result = m.g.modules[ms.position] + var ms = getModule(s) + result = m.g.modules[ms.position] proc initLoc(k: TLocKind, lode: PNode, s: TStorageLoc, flags: TLocFlags = {}): TLoc = result = TLoc(k: k, storage: s, lode: lode, diff --git a/compiler/options.nim b/compiler/options.nim index d6f67279774f..1003c2454771 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -685,7 +685,6 @@ proc usesWriteBarrier*(conf: ConfigRef): bool {.inline.} = conf.selectedGC >= gc template compilationCachePresent*(conf: ConfigRef): untyped = false -# conf.symbolFiles in {v2Sf, writeOnlySf} template optPreserveOrigSource*(conf: ConfigRef): untyped = optEmbedOrigSrc in conf.globalOptions From 85af5a7a899367c856f7b89a440481c3d1a293ee Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 30 Oct 2024 23:33:53 +0800 Subject: [PATCH 5/9] fixes nimsuggest --- nimsuggest/nimsuggest.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index 04bae08c1fd0..7e1355d128ac 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -234,7 +234,7 @@ proc clearInstCache(graph: ModuleGraph, projectFileIdx: FileIndex) = for tbl in mitems(graph.attachedOps): var attachedOpsToDelete = newSeq[ItemId]() for id in tbl.keys: - if id.module == projectFileIdx.int and sfOverridden in resolveAttachedOp(graph, tbl[id]).flags: + if id.module == projectFileIdx.int and sfOverridden in tbl[id].flags: attachedOpsToDelete.add id for id in attachedOpsToDelete: tbl.del id From 225c30f2f905f2fb59f7300944fe9a5d27754326 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 31 Oct 2024 20:55:48 +0800 Subject: [PATCH 6/9] remove ModuleIter --- compiler/importer.nim | 6 +++--- compiler/lookups.nim | 29 ++++++++++++++--------------- compiler/modulegraphs.nim | 24 ++++++------------------ compiler/nimeval.nim | 8 ++++---- compiler/sem.nim | 2 +- compiler/semcall.nim | 6 +++--- compiler/suggest.nim | 2 +- 7 files changed, 32 insertions(+), 45 deletions(-) diff --git a/compiler/importer.nim b/compiler/importer.nim index 829f31f19f72..c7128f5f2047 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -144,13 +144,13 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) = # for an enumeration we have to add all identifiers if multiImport: # for a overloadable syms add all overloaded routines - var it: ModuleIter = default(ModuleIter) - var e = initModuleIter(it, c.graph, fromMod, s.name) + var it: TIdentIter = default(TIdentIter) + var e = initIdentIter(it, selectTabs(c.graph, fromMod), s.name) while e != nil: if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3") if s.kind in ExportableSymKinds: rawImportSymbol(c, e, fromMod, importSet) - e = nextModuleIter(it, c.graph) + e = nextIdentIter(it, selectTabs(c.graph, fromMod)) else: rawImportSymbol(c, s, fromMod, importSet) suggestSym(c.graph, n.info, s, c.graph.usageSym, false) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index b6ecbe235f8b..18bcbd82e28e 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -109,9 +109,9 @@ proc localSearchInScope*(c: PContext, s: PIdent): PSym = scope = scope.parent result = strTableGet(scope.symbols, s) -proc initIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule; name: PIdent; +proc initIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule; name: PIdent; g: ModuleGraph): PSym = - result = initModuleIter(ti, g, im.m, name) + result = initIdentIter(ti, selectTabs(g, im.m), name) while result != nil: let b = case im.mode @@ -120,12 +120,12 @@ proc initIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule; n of importExcept: name.id notin im.exceptSet if b and not containsOrIncl(marked, result.id): return result - result = nextModuleIter(ti, g) + result = nextIdentIter(ti, selectTabs(g, im.m)) -proc nextIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule; +proc nextIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule; g: ModuleGraph): PSym = while true: - result = nextModuleIter(ti, g) + result = nextIdentIter(ti, selectTabs(g, im.m)) if result == nil: return nil case im.mode of importAll: @@ -139,7 +139,7 @@ proc nextIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule; return result iterator symbols(im: ImportedModule; marked: var IntSet; name: PIdent; g: ModuleGraph): PSym = - var ti: ModuleIter = default(ModuleIter) + var ti: TIdentIter = default(TIdentIter) var candidate = initIdentIter(ti, marked, im, name, g) while candidate != nil: yield candidate @@ -330,7 +330,6 @@ type oimSymChoiceLocalLookup TOverloadIter* = object it*: TIdentIter - mit*: ModuleIter m*: PSym mode*: TOverloadIterMode symChoiceIndex*: int @@ -740,7 +739,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = scope = scope.parent if scope == nil: for i in 0..c.imports.high: - result = initIdentIter(o.mit, o.marked, c.imports[i], ident, c.graph) + result = initIdentIter(o.it, o.marked, c.imports[i], ident, c.graph) if result != nil: o.currentScope = nil o.importIdx = i @@ -767,7 +766,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = ident) o.mode = oimSelfModule else: - result = initModuleIter(o.mit, c.graph, o.m, ident) + result = initIdentIter(o.it, selectTabs(c.graph, o.m), ident) else: noidentError(c.config, n[1], n) result = errorSym(c, n[1]) @@ -801,7 +800,7 @@ proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym var idx = o.importIdx+1 o.importIdx = c.imports.len # assume the other imported modules lack this symbol too while idx < c.imports.len: - result = initIdentIter(o.mit, o.marked, c.imports[idx], o.it.name, c.graph) + result = initIdentIter(o.it, o.marked, c.imports[idx], o.it.name, c.graph) if result != nil: # oh, we were wrong, some other module had the symbol, so remember that: o.importIdx = idx @@ -812,7 +811,7 @@ proc symChoiceExtension(o: var TOverloadIter; c: PContext; n: PNode): PSym = result = nil assert o.currentScope == nil while o.importIdx < c.imports.len: - result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph) + result = initIdentIter(o.it, o.marked, c.imports[o.importIdx], o.it.name, c.graph) #while result != nil and result.id in o.marked: # result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx]) if result != nil: @@ -836,12 +835,12 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = else: o.importIdx = 0 if c.imports.len > 0: - result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph) + result = initIdentIter(o.it, o.marked, c.imports[o.importIdx], o.it.name, c.graph) if result == nil: result = nextOverloadIterImports(o, c, n) break elif o.importIdx < c.imports.len: - result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph) + result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx], c.graph) if result == nil: result = nextOverloadIterImports(o, c, n) else: @@ -849,7 +848,7 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = of oimSelfModule: result = nextIdentIter(o.it, c.topLevelScope.symbols) of oimOtherModule: - result = nextModuleIter(o.mit, c.graph) + result = nextIdentIter(o.it, selectTabs(c.graph, o.m)) of oimSymChoice: if o.symChoiceIndex < n.len: result = n[o.symChoiceIndex].sym @@ -890,7 +889,7 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = incl o.marked, result.id elif o.importIdx < c.imports.len: - result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph) + result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx], c.graph) #assert result.id notin o.marked #while result != nil and result.id in o.marked: # result = nextIdentIter(o.it, c.imports[o.importIdx]) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 307871895af0..c4b5eb40eb85 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -11,7 +11,7 @@ ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -import std/[intsets, tables, hashes, strtabs, algorithm, os, strutils, parseutils] +import std/[intsets, tables, hashes, strtabs, os, strutils, parseutils] import ../dist/checksums/src/checksums/md5 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages, suggestsymdb @@ -189,20 +189,8 @@ proc strTableAdds*(g: ModuleGraph, m: PSym, s: PSym) = strTableAdd(semtab(g, m), s) strTableAdd(semtabAll(g, m), s) -type - ModuleIter* = object - modIndex: int - ti: TIdentIter - importHidden: bool - -proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym = - assert m.kind == skModule - mi.modIndex = m.position - mi.importHidden = optImportHidden in m.options - result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name) - -proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym = - result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden)) +template selectTabs*(g: ModuleGraph; m: PSym): TStrTable = + g.ifaces[m.position].interfSelect(optImportHidden in m.options) iterator allSyms*(g: ModuleGraph; m: PSym): PSym = let importHidden = optImportHidden in m.options @@ -226,11 +214,11 @@ proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym = result = someSym(g, g.systemModule, name) iterator systemModuleSyms*(g: ModuleGraph; name: PIdent): PSym = - var mi: ModuleIter = default(ModuleIter) - var r = initModuleIter(mi, g, g.systemModule, name) + var ti: TIdentIter = default(TIdentIter) + var r = initIdentIter(ti, selectTabs(g, g.systemModule), name) while r != nil: yield r - r = nextModuleIter(mi, g) + r = nextIdentIter(ti, selectTabs(g, g.systemModule)) iterator typeInstCacheItems*(g: ModuleGraph; s: PSym): PType = if g.typeInstCache.contains(s.itemId): diff --git a/compiler/nimeval.nim b/compiler/nimeval.nim index 0833cfeb32bf..0b6f40fc2d7c 100644 --- a/compiler/nimeval.nim +++ b/compiler/nimeval.nim @@ -9,7 +9,7 @@ ## exposes the Nim VM to clients. import - ast, modules, condsyms, + ast, astalgo, modules, condsyms, options, llstream, lineinfos, vm, vmdef, modulegraphs, idents, pathutils, scriptconfig, std/[compilesettings, tables, os] @@ -40,14 +40,14 @@ proc selectUniqueSymbol*(i: Interpreter; name: string; assert i != nil assert i.mainModule != nil, "no main module selected" let n = getIdent(i.graph.cache, name) - var it: ModuleIter = default(ModuleIter) - var s = initModuleIter(it, i.graph, i.mainModule, n) + var it: TIdentIter = default(TIdentIter) + var s = initIdentIter(it, selectTabs(i.graph, i.mainModule), n) result = nil while s != nil: if s.kind in symKinds: if result == nil: result = s else: return nil # ambiguous - s = nextModuleIter(it, i.graph) + s = nextIdentIter(it, selectTabs(i.graph, i.mainModule)) proc selectRoutine*(i: Interpreter; name: string): PSym = ## Selects a declared routine (proc/func/etc) from the main module. diff --git a/compiler/sem.nim b/compiler/sem.nim index a116a28db11b..dc842644a7cc 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -18,7 +18,7 @@ import evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity, lowerings, plugins/active, lineinfos, int128, isolation_check, typeallowed, modulegraphs, enumtostr, concepts, astmsgs, - extccomp, layeredtable + layeredtable import vtables import std/[strtabs, math, tables, intsets, strutils, packedsets] diff --git a/compiler/semcall.nim b/compiler/semcall.nim index fb2f9e97a78e..435730dee14c 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -93,14 +93,14 @@ proc addTypeBoundSymbols(graph: ModuleGraph, arg: PType, name: PIdent, if t != nil and t.owner.kind == skModule: # search module for routines attachable to `t` let module = t.owner - var iter = default(ModuleIter) - var s = initModuleIter(iter, graph, module, name) + var iter = default(TIdentIter) + var s = initIdentIter(iter, selectTabs(graph, module), name) while s != nil: if s.kind in filter and s.isAttachableRoutineTo(t) and not containsOrIncl(marker, s.id): # least priority scope, less than explicit imports: syms.add((s, -2)) - s = nextModuleIter(iter, graph) + s = nextIdentIter(iter, selectTabs(graph, module)) proc pickBestCandidate(c: PContext, headSymbol: PNode, n, orig: PNode, diff --git a/compiler/suggest.nim b/compiler/suggest.nim index a5213086bdb0..2f54852809a4 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -35,7 +35,7 @@ import prefixmatches, suggestsymdb from wordrecg import wDeprecated, wError, wAddr, wYield -import std/[algorithm, sets, parseutils, tables] +import std/[algorithm, sets, parseutils] when defined(nimsuggest): import pathutils # importer From c58569fa319334566faedd34e2f2388190e6f5fb Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 31 Oct 2024 21:00:04 +0800 Subject: [PATCH 7/9] appease old csources --- compiler/modulegraphs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index c4b5eb40eb85..27f02a10f90f 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -190,7 +190,7 @@ proc strTableAdds*(g: ModuleGraph, m: PSym, s: PSym) = strTableAdd(semtabAll(g, m), s) template selectTabs*(g: ModuleGraph; m: PSym): TStrTable = - g.ifaces[m.position].interfSelect(optImportHidden in m.options) + interfSelect(g.ifaces[m.position], optImportHidden in m.options) iterator allSyms*(g: ModuleGraph; m: PSym): PSym = let importHidden = optImportHidden in m.options From 96095d00bad307d7f4117c291a3eb636548ddb12 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 31 Oct 2024 21:19:59 +0800 Subject: [PATCH 8/9] Apply suggestions from code review --- compiler/ast.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 00779a30965f..5ff986776c3e 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -834,7 +834,7 @@ var gconfig {.threadvar.}: Gconfig proc setUseIc*(useIc: bool) = gconfig.useIc = useIc proc comment*(n: PNode): string = - if nfHasComment in n.flags: + if nfHasComment in n.flags and not gconfig.useIc: # IC doesn't track comments, see `packed_ast`, so this could fail result = gconfig.comments[n.nodeId] else: From 4b4994e4c45641450632986321c822397dff9ff2 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 1 Nov 2024 18:17:22 +0800 Subject: [PATCH 9/9] rememberExpansion --- compiler/sem.nim | 1 + compiler/semdata.nim | 15 ++++++++++++++- compiler/semexprs.nim | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index dc842644a7cc..c9479342e992 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -505,6 +505,7 @@ const proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym, flags: TExprFlags = {}; expectedType: PType = nil): PNode = + rememberExpansion(c, nOrig.info, sym) pushInfoContext(c.config, nOrig.info, sym.detailedInfo) let info = getCallLineInfo(n) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index cb80bda6aa0d..05cbd21e3c63 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -573,4 +573,17 @@ proc addToGenericProcCache*(c: PContext; s: PSym; inst: PInstantiation) = c.graph.procInstCache.mgetOrPut(s.itemId, @[]).add inst proc addToGenericCache*(c: PContext; s: PSym; inst: PType) = - c.graph.typeInstCache.mgetOrPut(s.itemId, @[]).add inst \ No newline at end of file + c.graph.typeInstCache.mgetOrPut(s.itemId, @[]).add inst + +proc storeExpansion(c: PContext; info: TLineInfo; expandedSym: PSym) = + discard + +proc rememberExpansion*(c: PContext; info: TLineInfo; expandedSym: PSym) = + ## Templates and macros are very special in Nim; these have + ## inlining semantics so after semantic checking they leave no trace + ## in the sem'checked AST. This is very bad for IDE-like tooling + ## ("find all usages of this template" would not work). We need special + ## logic to remember macro/template expansions. This is done here and + ## delegated to the "rod" file mechanism. + if c.config.symbolFiles != disabledSf: + storeExpansion(c, info, expandedSym) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 68e9c35a5daa..c5fbbe8a4bd3 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -27,6 +27,7 @@ const proc semTemplateExpr(c: PContext, n: PNode, s: PSym, flags: TExprFlags = {}; expectedType: PType = nil): PNode = + rememberExpansion(c, n.info, s) let info = getCallLineInfo(n) markUsed(c, info, s) onUse(info, s)