Skip to content

Commit

Permalink
improv tests
Browse files Browse the repository at this point in the history
  • Loading branch information
ailrst committed Sep 16, 2024
1 parent f7fd896 commit 87988f5
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 36 deletions.
6 changes: 6 additions & 0 deletions examples/csmith/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
*.out
*.gts
*.adt
*.c
*.relf
*.bir
71 changes: 71 additions & 0 deletions examples/csmith/gen.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import subprocess


fn_min = 10
fn_max = 11
block_min = 4
block_max = 8
seed_min = 0
num_rand = 50

# produces programs that we can't handle
# cmd = ["csmith", "--seed", str(seed), "--max-block-size", str(max_blocksize), "--max-funcs", str(max_funcs), "--output", f"{fname}.c"]

csmith_safe = "--no-arrays --no-bitfields --no-checksum --no-comma-operators --no-longlong --no-int8 --no-uint8 --no-float --no-math64 --no-inline-function --no-safe-math --no-packed-struct --no-pointers --no-structs --no-unions --no-volatile-pointers --no-const-pointers".split(" ")

files = []
vals = {}

for globals in ["--no-global-variables", "--global-variables"]:
for max_funcs in range(fn_min, fn_max):
for seed in range(seed_min, seed_min + num_rand):
for max_blocksize in range(block_min, block_max):
fname = f"c-{max_funcs}-{max_blocksize}-{seed}-{globals.replace('-', '')}"
files.append(fname)
vals[fname] = {
'globals' : globals,
'max_funcs': max_funcs,
'seed' : seed,
'max_blocksize': max_blocksize
}


def csmith():
for fname in files:
globals = vals[fname]['globals']
seed = vals[fname]['seed']
max_funcs = vals[fname]['max_funcs']
max_blocksize = vals[fname]['max_blocksize']
cmd = ["csmith", "--max-block-size", str(max_blocksize), "--max-funcs", str(max_funcs), "--output", f"{fname}.c"] + csmith_safe
print(cmd)
gen = subprocess.run(cmd)
checksum_cmd = f"gcc {fname}.c -o ${fname}-native.out && ./{fname}.out && rm {fname}-native.out"


def checksum():
for fname in files:
print(fname)
checksum_cmd = f"clang {fname}.c -w -o {fname}-native.out -I /usr/include/csmith && ./{fname}-native.out && rm {fname}-native.out"
print(checksum_cmd)
try:
checksum = subprocess.run(checksum_cmd, capture_output=True, text=True, shell=True, timeout=10)
print("checksum" , checksum.stdout, checksum.stderr)
with open(fname + ".checksum", 'w') as f:
f.write(checksum.stdout)
except Exception as e:
print(e)

def lift():
for fname in files:
compilecmd = ['clang-15', '-w', '-target', 'aarch64-linux-gnu', fname + ".c", '-I', '/usr/include/csmith', '-o', fname + '.out']
print(compilecmd)
compile = subprocess.run(compilecmd)
cmd = ["bap", f"{fname}.out", "-d", "adt:" + fname + ".adt", "-d", "bir:"+ fname + ".bir" ]
relfcmd = f"readelf -s -r -w {fname}.out > {fname}.relf"
print(cmd)
subprocess.run(cmd)
print(relfcmd)
subprocess.run(relfcmd, shell=True)

csmith()
lift()
80 changes: 44 additions & 36 deletions src/test/scala/TimeAnalysis.scala
Original file line number Diff line number Diff line change
Expand Up @@ -75,67 +75,72 @@ class TimeStaticAnalysis extends AnyFunSuite, TestUtil {
}
}


val examples = getFiles("examples/csmith").filter(c => c.endsWith(".adt")).map(_.stripSuffix(".adt"))
Logger.setLevel(LogLevel.INFO)



info("Config")
def config(name: String) = ILLoadingConfig("examples/csmith/" + name + ".adt", "examples/csmith/" + name + ".relf")
// use filesize to approximate size of program and sort tests by size
def map2nd[A,B,C](a: Iterable[(C, A)], f: A => B) = a.map((x: (C, A)) => (x._1, f(x._2)))
val loads = examples.map(c => (c, config(c))).toList
val ctx = map2nd(loads, x => {
try {
IRLoading.load(x)
} catch {
case e => {
Logger.error(s"$e at ${x.inputFile}")
throw e
}
}

val loads2 = loads.map(x => {
val (c,cfg) = x
val chars = (File(cfg.inputFile)).length
(c,cfg,chars)
})
val cleanup = map2nd(ctx, IRTransform.doCleanup)
val loads3 = loads2.sortBy(_._3).take(50).map(c => (c._1, c._2))
Logger.info(loads2.map(c => s"${c._2.inputFile} ${c._3}").mkString("\n"))

val complexity = map2nd(cleanup, c => (c, ProgStats.get(c.program))).toMap
val sorted = complexity.toList.sortBy(c => c._2._2.blocks)
Logger.warn(sorted.map(c => s"${c._1}, ${c._2._2}").mkString("\n"))
val sortedcontexts : List[(String, IRContext)] = sorted.map(c => (c._1, c._2._1))
val sorted = loads3

def doAnalysis(ctx: IRContext) : Future[List[(String, String, Long)]] = Future {
Logger.warn(sorted.map(c => s"${c._1}, ${c._2}").mkString("\n"))

def doAnalysis(ex: String) : Future[(ProgStats, List[(String, String, Long)])] = Future {
blocking {
StaticAnalysis.analyse(ctx, StaticAnalysisConfig(), 0).timer.checkPoints()
// load again and analyse
val c = IRLoading.load(config(ex))
val ctx = IRTransform.doCleanup(c)
val comp = ProgStats.get(c.program)
(comp, StaticAnalysis.analyse(ctx, StaticAnalysisConfig(), 0).timer.checkPoints())
}
}
var stop = false
val result : List[(String, List[(String, String, Long)])] = sortedcontexts.map(v => {
val (testn,ctx) = v
// give up after \timeout_thresh consecutive timeouts
val timeout_thresh = 3
var timeouts = 0
val result : List[(String, ProgStats, List[(String, String, Long)])] = sorted.map(v => {
val (testn,cfg) = v
try {
if stop then {
(testn, List((s"Timeout triggered stop", "", 0 : Long)))
if (timeouts >= timeout_thresh) then {
None
} else {
Logger.warn(s"TESTING $testn")
val comp = complexity(testn)._2
val (comp,r) = Await.result(doAnalysis(testn), 240000.millis)
Logger.warn(comp)
val r = Await.result(doAnalysis(ctx), 120000.millis)
Logger.warn("CHECKPOINTS:")
Logger.warn(r.map(c => s"${c._1},${c._2},${c._3}").mkString("\n"))
(testn, r)
timeouts = 0
Some((testn, comp, r))
}
} catch {
case e : scala.concurrent.TimeoutException => {
timeouts += 1
Logger.error(e)
stop = true
(testn, List((s"$e", "", 0 : Long)))
None
}
case e => {
Logger.error(e, e.getStackTrace.take(10).mkString("\n"))
(testn, List((s"$e", "", 0 : Long)))
None
}
}
}).toList
// test filename ,statements, blocks, procedures, checkpoint name, checkpoint loc , time delta
val times = result.flatMap(x => x._2.map((checkpoint : (String, String, Long)) => {
val comp = complexity(x._1)._2
}).collect {
case Some(x) => x
}.toList
// test filename , statements, blocks, procedures, checkpoint name, checkpoint loc , time delta
val times = result.flatMap(x => x._3.map((checkpoint : (String, String, Long)) => {
val comp = x._2
(x._1, comp.statements,comp.blocks,comp.procedures,
checkpoint._1,
checkpoint._3
Expand All @@ -154,11 +159,14 @@ class TimeStaticAnalysis extends AnyFunSuite, TestUtil {
}

test("Getexamples") {
Logger.setLevel(LogLevel.WARN)

val r = examples().sortBy(x => x._5)
val grouped = r.groupBy(x => x._5).filter(i => !i._1.contains("Timeout") && !i._1.contains("Exception"))

var plotfile = "set terminal 'svg'; set output 'analysisres.svg' ; set xlabel \"statement count\" ; set ylabel \"analysis time (ms)\""
val outputPathPrefix = "src/test/analysisTiming"
var plotfile = s"set terminal 'svg' enhanced background rgb 'white'; set output '${outputPathPrefix}/analysisres.svg' ; set xlabel \"statement count (log scale)\" ; set ylabel \"analysis time (ms) (log scale)\"\nset logscale x 2\nset logscale y 2"


var plotcmds = List[String]()

Expand All @@ -168,16 +176,16 @@ class TimeStaticAnalysis extends AnyFunSuite, TestUtil {
val y = vs._6 // time
s"$x $y"
})).mkString("\n")
val pname = s"dat/${n}.dat"
val pname = s"${outputPathPrefix}/${n}.dat"
log(pname, table)
val plotcmd = s"'${pname}' title \"${n}\" with lines"
plotcmds = plotcmd::plotcmds
}
val pl = s"plot ${plotcmds.mkString(", ")}"
val gp = plotfile + "\n" + pl
println(gp)
log("dat/plot.gp", gp)
Seq("gnuplot", "dat/plot.gp").!!
log(outputPathPrefix + "/plot.gp", gp)
Seq("gnuplot", outputPathPrefix + "/plot.gp").!!


val csv = r.map(c => c.toList.mkString(",")).mkString("\n")
Expand Down

0 comments on commit 87988f5

Please sign in to comment.