Skip to content

Commit

Permalink
Update keras model, working iffy with insertion
Browse files Browse the repository at this point in the history
  • Loading branch information
Dhvani Patel committed Jul 20, 2017
1 parent 63aac7c commit dad65c4
Show file tree
Hide file tree
Showing 6 changed files with 68 additions and 61 deletions.
Binary file modified __pycache__/toCheck.pypy-41.pyc
View file
Binary file not shown.
33 changes: 19 additions & 14 deletions keras_model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@

def getInputTen():
one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, _, _, _, _, _ = perform(0)
print type(one_hot_good)
print one_hot_good
windowInd = 0
fileInd = 0
batchInd = 1
Expand Down Expand Up @@ -141,6 +143,8 @@ def getInputTen():

def getOutputTen():
_, _, _, _, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, _ = perform(0)
print type(one_hot_good_out)
print one_hot_good_out
windowInd = 0
fileInd = 0
batchInd = 1
Expand Down Expand Up @@ -171,7 +175,7 @@ def getOutputTen():
else:
print "not here"
toPassOne.append(bruhOne)
print bruhOne
#print bruhOne

toPassTwo = []
for x in range(10):
Expand Down Expand Up @@ -274,7 +278,7 @@ def getOutputTen():
#print toPassOne[1]
a = numpy.array(toPass)
#a = b[None, :]
print a.shape
#print a.shape
#count+=1
#print "COUNT"
#print count
Expand Down Expand Up @@ -313,9 +317,9 @@ def getOutputTen():


def getInputValTen():
one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, _, _, _, _, _ = perform(1001)
one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, _, _, _, _, _ = perform(1002)
windowInd = 0
fileInd = 1001
fileInd = 1002
batchInd = 1
#count = 0
while fileInd <= 2000: # 462540
Expand Down Expand Up @@ -422,9 +426,9 @@ def getInputValTen():
one_hot_bad_sub.insert(p, old_one_hot_bad_sub[len(old_one_hot_bad_sub)-numBadSubLeft+p])

def getOutputValTen():
_, _, _, _, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, _ = perform(1001)
_, _, _, _, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, _ = perform(1002)
windowInd = 0
fileInd = 1001
fileInd = 1002
batchInd = 1
while fileInd <= 2000: # 462540
#while windowInd < int(len(insArr)/10):
Expand Down Expand Up @@ -453,7 +457,7 @@ def getOutputValTen():
else:
print "not here"
toPassOne.append(bruhOne)
print bruhOne
#print bruhOne

toPassTwo = []
for x in range(10):
Expand Down Expand Up @@ -556,7 +560,7 @@ def getOutputValTen():
#print toPassOne[1]
a = numpy.array(toPass)
#a = b[None, :]
print a.shape
#print a.shape
#count+=1
#print "COUNT"
#print count
Expand Down Expand Up @@ -595,14 +599,15 @@ def getOutputValTen():


def getInputTestTen():
one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, _, _, _, _, passInsErrorInd = perform(2077)
fileInd = 2077
one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, _, _, _, _, passInsErrorInd = perform(2037)
fileInd = 2037
batchInd = 1

if True:
if True:
print passInsErrorInd
#print passInsErrorInd
print "ERROR IND"
print one_hot_good
toPassOne = []
for x in range(10):
y = x
Expand Down Expand Up @@ -632,7 +637,7 @@ def getInputTestTen():
#print len(toPassFour)
#toPass = np.array((toPassOne, toPassTwo, toPassThree, toPassFour))
#print toPass.shape
toPass = toPassOne[:]
toPass = toPassTwo[:]
a = numpy.array(toPass).astype(int)
#print b.shape
return a
Expand Down Expand Up @@ -784,8 +789,8 @@ def initData():
opt = optimizers.SGD(lr=0.001, momentum=0.005)
#opt = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-08, decay=0.0)
#opt = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
model.compile(loss = "categorical_crossentropy", optimizer = opt, metrics=['accuracy'])
#model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
#model.compile(loss = "categorical_crossentropy", optimizer = opt, metrics=['accuracy'])
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

#zipped = iter()
#print type(zipped)
Expand Down
58 changes: 29 additions & 29 deletions mutate_token_insert.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,10 @@ def insertTokMutS(raw_tokens, all_tokens, raw_text):

from py_mutations_hub import getRid
#print "OKAY"
print len(raw_tokens_pass)
#print len(raw_tokens_pass)
#test = getRid(raw_tokens_pass, True)
#print len(test)
print len(actual_token_len)
#print len(actual_token_len)
#print type(radha)

num_lines = len(actual_token_len)
Expand All @@ -86,20 +86,20 @@ def insertTokMutS(raw_tokens, all_tokens, raw_text):
insToks = []
chosens = []

print numTokensNeeded
print "import num"
#print numTokensNeeded
#print "import num"
haha = -1
radOut = 0
while radOut < numTokensNeeded:
#chosen = raw_tokens_pass[chosenLineInd]

chosenInd = randint(0,84)
chosenToken = data["indexes_m"][chosenInd]
print "RAD S"
print radOut
print len(chosens)
print len(insToks)
print "RAD O"
#print "RAD S"
#print radOut
#print len(chosens)
#print len(insToks)
#print "RAD O"

global new_token
new_token = []
Expand All @@ -121,9 +121,9 @@ def insertTokMutS(raw_tokens, all_tokens, raw_text):
param_end = param_start + 9
haha = param_end
chosenLineInd = randint(param_start, param_end) #num_lines-1
print "inds"
print chosenLineInd
print "stop"
#print "inds"
#print chosenLineInd
#print "stop"
#chosen = raw_tokens_pass[chosenLineInd]
chosen = actual_token_len[chosenLineInd]
#chosen = Token(tokenize.tok_name[raw_tokens_pass[chosenLineInd][0]], raw_tokens_pass[chosenLineInd][1], raw_tokens_pass[chosenLineInd][2][0], raw_tokens_pass[chosenLineInd][2][1], raw_tokens_pass[chosenLineInd][3][0], raw_tokens_pass[chosenLineInd][3][1], raw_tokens_pass[chosenLineInd][4])
Expand All @@ -136,18 +136,18 @@ def insertTokMutS(raw_tokens, all_tokens, raw_text):
#print raw_tokens_pass[chosenLineInd][0]

#print all_tokens[chosenLineInd].value
print num_lines - 1
print len(raw_tokens_pass)
print len(actual_token_len)
#print num_lines - 1
#print len(raw_tokens_pass)
#print len(actual_token_len)
#print raw_tokens_pass[50][4]
print actual_token_len[chosenLineInd].line
#print actual_token_len[chosenLineInd].line
#print raw_text
toAddBeforeInd = source_code.index(actual_token_len[chosenLineInd].line)
print toAddBeforeInd
#print toAddBeforeInd

temp = source_code[toAddBeforeInd:toAddBeforeInd+len(actual_token_len[chosenLineInd].line)]
print temp
print actual_token_len[chosenLineInd].value
#print temp
#print actual_token_len[chosenLineInd].value
#print "kobe"
#print raw_tokens_pass[chosenLineInd][1]

Expand Down Expand Up @@ -207,7 +207,7 @@ def insertTokMutS(raw_tokens, all_tokens, raw_text):
radOut = radOut
insToks.remove(insTok)
chosens.remove(chosen)
print "test_t"
#print "test_t"
else:
radOut = radOut + 1

Expand All @@ -223,10 +223,10 @@ def insertTokMutS(raw_tokens, all_tokens, raw_text):
'''
#print NAH
#print "NAH"
print ".___."
print len(chosens)
print len(insToks)
print "NAH"
#print ".___."
#print len(chosens)
#print len(insToks)
#print "NAH"

return new_text, NO_TOKEN, INSERTION, out_tokens_loc, chosens, insToks

Expand Down Expand Up @@ -279,8 +279,8 @@ def insertTokMut(raw_tokens, raw_text):
insToks = []
chosens = []

print numTokensNeeded
print "import num"
#print numTokensNeeded
#print "import num"
for rad in range(numTokensNeeded):

#chosen = raw_tokens_pass[chosenLineInd]
Expand Down Expand Up @@ -308,9 +308,9 @@ def insertTokMut(raw_tokens, raw_text):
param_end = param_start + 9

chosenLineInd = randint(param_start, param_end) #num_lines-1
print "inds"
print chosenLineInd
print "stop"
#print "inds"
#print chosenLineInd
#print "stop"
chosen = Token(tokenize.tok_name[raw_tokens_pass[chosenLineInd][0]], raw_tokens_pass[chosenLineInd][1], raw_tokens_pass[chosenLineInd][2][0], raw_tokens_pass[chosenLineInd][2][1], raw_tokens_pass[chosenLineInd][3][0], raw_tokens_pass[chosenLineInd][3][1], raw_tokens_pass[chosenLineInd][4])
chosens.append(chosen)

Expand Down
Binary file modified mutate_token_insert.pyc
View file
Binary file not shown.
38 changes: 20 additions & 18 deletions py_mutations_hub.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,9 +310,9 @@ def perform(curr):
allGood = []
global all_tokens
allGood = all_tokens[:]
print "come on"
print len(all_tokens)
print len(allGood)
#print "come on"
#print len(all_tokens)
#print len(allGood)
one_hot_good = vocabularize_tokens(all_tokens, False)
one_hot_good_out = []
for x in range(len(all_tokens)+(WINDOW_SIZE-1)+(WINDOW_SIZE-1)):
Expand Down Expand Up @@ -345,9 +345,9 @@ def perform(curr):
#print len(passToks)
#print "come
global all_tokens
print "dhadha"
print len(all_tokens)
print len(allGood)
#print "dhadha"
#print len(all_tokens)
#print len(allGood)
passBall = allGood[:]
new_i_text, NO_TOKEN, INSERTION, out_tokens_loc, chosenTrueLineInds, insToks = insertTokMutS(raw_tokens, passBall, source_code)

Expand All @@ -357,8 +357,8 @@ def perform(curr):
break

new_tokens_ins = allGood[:]
print "BOL BOL BOL"
print len(new_tokens_ins)
#print "BOL BOL BOL"
#print len(new_tokens_ins)
temp = insToks[:]
for insTok in temp:
if insTok.type == "NL":
Expand All @@ -369,8 +369,8 @@ def perform(curr):
vocab_entry = open_closed_tokens(chosenTrueLineInd)
chosenTrueLineInds[chosenTrueLineInds.index(chosenTrueLineInd)].value = vocab_entry
#print vocab_entry
print "OK ------------------------------"
print len(new_tokens_ins)
#print "OK ------------------------------"
#print len(new_tokens_ins)
#print len(chosenTrueLineInds)
#print len(all_tokens)
for wow in range(len(chosenTrueLineInds)):
Expand All @@ -390,14 +390,14 @@ def perform(curr):
#print bruhInd
#print "gotchu"
new_tokens_ins.insert(bruhInd+1, insTok)
print "START DEBUG"
print insTok.value
print len(new_tokens_ins)
print new_tokens_ins[bruhInd+1].value
#print "START DEBUG"
#print insTok.value
#print len(new_tokens_ins)
#print new_tokens_ins[bruhInd+1].value

one_hot_bad_ins = vocabularize_tokens(new_tokens_ins, True)
#print one_hot_bad_ins[bruhInd+1+WINDOW_SIZE-1]
print "DONE DEBUG"
#print "DONE DEBUG"

#print len(new_tokens_ins)
#print len(one_hot_bad_ins)
Expand Down Expand Up @@ -659,14 +659,16 @@ def perform(curr):
return one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out,passInsErrorInd

else:
#print "Try again..."
print "Try again..."
print curr
#print all_rows[curr][0]
return 1, None, None, None, 1, None, None, None. None
return 1, None, None, None, 1, None, None, None, None


if __name__ == '__main__':
perform(77)
perform(0)
print "done one"
perform(0)
sys.exit()
for x in range(3):
# 36, 80, 124, 126, 177
Expand Down
Binary file modified py_mutations_hub.pyc
View file
Binary file not shown.

0 comments on commit dad65c4

Please sign in to comment.