diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..d4c48ab
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+train_pre_data.txt
diff --git a/__pycache__/toCheck.pypy-41.pyc b/__pycache__/toCheck.pypy-41.pyc
index 7002a44..61daed2 100644
Binary files a/__pycache__/toCheck.pypy-41.pyc and b/__pycache__/toCheck.pypy-41.pyc differ
diff --git a/keras_model.py b/keras_model.py
index 37543ae..fd5790d 100644
--- a/keras_model.py
+++ b/keras_model.py
@@ -17,7 +17,7 @@
from itertools import izip
import os
-
+import cPickle
import matplotlib.pyplot as plt
# BATCH = 60
@@ -34,10 +34,11 @@
BATCH_SIZE = 66
-def getInputTen():
- one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, _, _, _, _, _ = perform(0)
+def getInputTen(allTrainData):
+ #one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, _, _, _, _, _ = perform(0)
+ one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub = allTrainData[0][0],allTrainData[0][1], allTrainData[0][2], allTrainData[0][3]
while(one_hot_good == 1):
- one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, _, _, _, _, _ = perform(0)
+ one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub = allTrainData[0][0],allTrainData[0][1], allTrainData[0][2], allTrainData[0][3]
#print type(one_hot_good)
#print one_hot_good
windowInd = 0
@@ -150,10 +151,11 @@ def getInputTen():
#print "FILE IND"
print fileInd
windowInd = 0
- one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, _, _, _, _, _ = perform(fileInd)
+ #one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, _, _, _, _, _ = perform(fileInd)
+ one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub = allTrainData[fileInd][0],allTrainData[fileInd][1], allTrainData[fileInd][2], allTrainData[fileInd][3]
while(one_hot_good == 1):
fileInd+=1
- one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, _, _, _, _, _ = perform(fileInd)
+ one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub = allTrainData[fileInd][0],allTrainData[fileInd][1], allTrainData[fileInd][2], allTrainData[fileInd][3]
for p in range(numGoodLeft):
@@ -208,10 +210,13 @@ def getInputTen():
one_hot_bad_sub.insert(p, old_one_hot_bad_sub[len(old_one_hot_bad_sub)-numBadSubLeft+p])
'''
-def getOutputTen():
- _, _, _, _, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, _ = perform(0)
+def getOutputTen(allTrainData):
+ #_, _, _, _, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, _ = perform(0)
+ one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out = allTrainData[0][4],allTrainData[0][5], allTrainData[0][6], allTrainData[0][7]
+
while(one_hot_good_out == 1):
- _, _, _, _, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, _ = perform(0)
+ #_, _, _, _, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, _ = perform(0)
+ one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out = allTrainData[0][4],allTrainData[0][5], allTrainData[0][6], allTrainData[0][7]
#print type(one_hot_good_out)
#print one_hot_good_out
windowInd = 0
@@ -382,10 +387,12 @@ def getOutputTen():
fileInd += 1
windowInd = 0
- _, _, _, _, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, _ = perform(fileInd)
+ #_, _, _, _, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, _ = perform(fileInd)
+ one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out = allTrainData[fileInd][4],allTrainData[fileInd][5], allTrainData[fileInd][6], allTrainData[fileInd][7]
while(one_hot_good_out == 1):
fileInd+=1
- _, _, _, _, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, _ = perform(fileInd)
+ #_, _, _, _, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, _ = perform(fileInd)
+ one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out = allTrainData[fileInd][4],allTrainData[fileInd][5], allTrainData[fileInd][6], allTrainData[fileInd][7]
#for p in range(numGoodOutLeft):
# one_hot_good_out.insert(p, old_one_hot_good_out[len(old_one_hot_good_out)-numGoodOutLeft+p])
@@ -945,8 +952,14 @@ def initData():
print "Start..."
#main_input = Input(shape=(10,87), dtype='int32', name='main_input')
+ allTrainData = cPickle.load( open( "train_pre_data.txt", "rb" ) )
+ print "GOT DATA"
-
+ sum = 0
+ for x in allTrainData:
+ sum += len(x[2])
+ print sum
+ print "SUM"
model = Sequential()
model.add(Dense(4, activation='relu', input_shape=(10, 88), batch_size=66))
model.add(Dropout(0.5))
@@ -986,10 +999,10 @@ def initData():
# SECOnD THOUSAND TOKENS: 1353925
history = model.fit_generator(
- izip(getInputTen(), getOutputTen()),
+ izip(getInputTen(allTrainData), getOutputTen(allTrainData)),
steps_per_epoch=16521,
- validation_data=izip(getInputValTen(), getOutputValTen()),
- validation_steps=20513,
+ #validation_data=izip(getInputValTen(), getOutputValTen()),
+ #validation_steps=20513,
epochs=5,
verbose=2
)
diff --git a/py_mutations_hub.py b/py_mutations_hub.py
index ae124ac..17801bf 100644
--- a/py_mutations_hub.py
+++ b/py_mutations_hub.py
@@ -18,6 +18,7 @@
from mutate_token_sub import subTokMutS
import sys
import cPickle
+from scipy import sparse
# NUM TOTAL: 462 563
# ACTUAl: 462 540
@@ -49,7 +50,9 @@
START_TOKEN = ''
END_TOKEN = ''
-def one_hot(indexed_tokens):
+def one_hot_a(indexed_tokens):
+ print indexed_tokens
+ print len(indexed_tokens)
one_hot = []
nb_classes = 88
one_hot_targets = np.eye(nb_classes)[indexed_tokens]
@@ -62,6 +65,9 @@ def one_hot(indexed_tokens):
#one_hot.astype(int)
#print type(one_hot[0][0])
return one_hot
+
+def one_hot(indexed_tokens):
+ return indexed_tokens
def set_from_json(all_tokens, flag):
@@ -318,13 +324,14 @@ def perform(curr):
#print len(all_tokens)
#print len(allGood)
one_hot_good = vocabularize_tokens(all_tokens, False)
- one_hot_good_out = []
- for x in range(len(all_tokens)+(WINDOW_SIZE-1)+(WINDOW_SIZE-1)):
- toAdd = []
- toAdd = [0] * NUM_BITS_OUTPUT
- toAdd[0] = 0
- toAdd[1] = 1 # FIRST BIT (01) - INDICATE NO ERROR (1 because rest are 0 and so add up to 1)
- one_hot_good_out.append(toAdd)
+ #one_hot_good_out = []
+ #for x in range(len(all_tokens)+(WINDOW_SIZE-1)+(WINDOW_SIZE-1)):
+ #toAdd = []
+ #toAdd = [0] * NUM_BITS_OUTPUT
+ #toAdd[0] = 0
+ #toAdd[1] = 1 # FIRST BIT (01) - INDICATE NO ERROR (1 because rest are 0 and so add up to 1)
+ #toAdd = [1]
+ #one_hot_good_out.append(1)
#print "DHVANI"
@@ -412,35 +419,37 @@ def perform(curr):
#print "NEXT STEP...C"
- passInsErrorInd = (bruhInd+1)+(WINDOW_SIZE-1)
+ #passInsErrorInd = (bruhInd+1)+(WINDOW_SIZE-1)
- one_hot_bad_ins_out = []
- trueErrorInd = (bruhInd+1)+(WINDOW_SIZE-1)
+ #one_hot_bad_ins_out = []
+ #trueErrorInd = (bruhInd+1)+(WINDOW_SIZE-1)
# INSERT OUT_PUT
- iterNum = len(new_tokens_ins)+(WINDOW_SIZE-1)+(WINDOW_SIZE-1)
+ #iterNum = len(new_tokens_ins)+(WINDOW_SIZE-1)+(WINDOW_SIZE-1)
#print "divide"
#print trueErrorInd
#print iterNum
+ '''
for x in range(iterNum):
#if x <= trueErrorInd <= (x+trueErrorInd):
#if x <= trueErrorInd <= x+(WINDOW_SIZE-1):
if True:
# DIFF - ACTUAL ERROR
#print x
- toAdd = []
- toAdd = [0] * NUM_BITS_OUTPUT
- toAdd[0] = 1 # FIRST BIT (10) - INDICATE ERROR
- toAdd[1] = 0
- if NO_TOKEN != None:
- toAdd[2] = 0
- toAdd[3] = 1
- if INSERTION != None:
- toAdd[4] = 0
- toAdd[5] = 0
- toAdd[6] = 1
- toAdd[7] = 1
+ #toAdd = []
+ #toAdd = [0] * NUM_BITS_OUTPUT
+ #toAdd[0] = 1 # FIRST BIT (10) - INDICATE ERROR
+ #toAdd[1] = 0
+ #if NO_TOKEN != None:
+ # toAdd[2] = 0
+ # toAdd[3] = 1
+ #if INSERTION != None:
+ # toAdd[4] = 0
+ # toAdd[5] = 0
+ # toAdd[6] = 1
+ #toAdd[7] = 1
+ toAdd = [0,3,6,7]
one_hot_bad_ins_out.append(toAdd)
else:
toAdd = []
@@ -448,6 +457,7 @@ def perform(curr):
toAdd[0] = 1
toAdd[1] = 0 # FIRST BIT (01) - INDICATE NO ERROR (1 because rest are 0 and so add up to 1)
one_hot_bad_ins_out.append(toAdd)
+ '''
#print "Morning"
#print len(new_tokens_ins)
#print len(one_hot_bad_ins_out)
@@ -508,11 +518,11 @@ def perform(curr):
one_hot_bad_del = vocabularize_tokens(new_tokens_del, True)
- one_hot_bad_del_out = []
- trueErrorInd = (bruhInd)+(WINDOW_SIZE-1)
+ #one_hot_bad_del_out = []
+ #trueErrorInd = (bruhInd)+(WINDOW_SIZE-1)
# DELETE OUT_PUT
- iterNum = len(new_tokens_del)+(WINDOW_SIZE-1)+(WINDOW_SIZE-1)
+ #iterNum = len(new_tokens_del)+(WINDOW_SIZE-1)+(WINDOW_SIZE-1)
#print "divide"
#print len(send)
#print trueErrorInd
@@ -523,24 +533,26 @@ def perform(curr):
#oneH_ind_deleted = set_from_json_nonarr(send, True)
#print oneH_ind_deleted
#print "rad"
+ '''
for x in range(iterNum):
#if x <= trueErrorInd <= (x+trueErrorInd):
if True:
# DIFF - ACTUAL ERROR
#print x
- toAdd = []
- toAdd = [0] * NUM_BITS_OUTPUT
- toAdd[0] = 1 # FIRST BIT (10) - INDICATE ERROR
- toAdd[1] = 0
- if YES_TOKEN != None:
- toAdd[2] = 1
- toAdd[3] = 0
- if DELETION != None:
- toAdd[4] = 0
- toAdd[5] = 1
- toAdd[6] = 0
- toAdd[7] = 1
- toAdd[17] = 1
+ #toAdd = []
+ #toAdd = [0] * NUM_BITS_OUTPUT
+ #toAdd[0] = 1 # FIRST BIT (10) - INDICATE ERROR
+ #toAdd[1] = 0
+ #if YES_TOKEN != None:
+ # toAdd[2] = 1
+ # toAdd[3] = 0
+ #if DELETION != None:
+ # toAdd[4] = 0
+ # toAdd[5] = 1
+ # toAdd[6] = 0
+ #toAdd[7] = 1
+ #toAdd[17] = 1
+ toAdd = [0,2,5,7,17]
one_hot_bad_del_out.append(toAdd)
else:
toAdd = []
@@ -548,6 +560,7 @@ def perform(curr):
toAdd[0] = 0
toAdd[1] = 1 # FIRST BIT (01) - INDICATE NO ERROR (1 because rest are 0 and so add up to 1)
one_hot_bad_del_out.append(toAdd)
+ '''
#print "Morning"
#print len(allGood)
#print len(one_hot_bad_del_out)
@@ -624,10 +637,10 @@ def perform(curr):
one_hot_bad_sub = vocabularize_tokens(new_tokens_sub, True)
- one_hot_bad_sub_out = []
- trueErrorInd = (bruhInd)+(WINDOW_SIZE-1)
+ #one_hot_bad_sub_out = []
+ #trueErrorInd = (bruhInd)+(WINDOW_SIZE-1)
# SUB OUT_PUT
- iterNum = len(new_tokens_sub)+(WINDOW_SIZE-1)+(WINDOW_SIZE-1)
+ #iterNum = len(new_tokens_sub)+(WINDOW_SIZE-1)+(WINDOW_SIZE-1)
#print "divide"
#print len(send)
#print trueErrorInd
@@ -638,26 +651,29 @@ def perform(curr):
#oneH_sub_switch = set_from_json_nonarr(sendS, True)
#print oneH_sub_switch
#print "rad"
+
+ '''
for x in range(iterNum):
#if x <= trueErrorInd <= (x+trueErrorInd):
#if x <= trueErrorInd <= x+(WINDOW_SIZE-1):
if True:
# DIFF - ACTUAL ERROR
#print x
- toAdd = []
- toAdd = [0] * NUM_BITS_OUTPUT
- toAdd[0] = 1 # FIRST BIT (10) - INDICATE ERROR
- toAdd[1] = 0
+ #toAdd = []
+ #toAdd = [0] * NUM_BITS_OUTPUT
+ #toAdd[0] = 1 # FIRST BIT (10) - INDICATE ERROR
+ #toAdd[1] = 0
- toAdd[2] = 1
- toAdd[3] = 0
+ #toAdd[2] = 1
+ #toAdd[3] = 0
- toAdd[4] = 1
- toAdd[5] = 0
- toAdd[6] = 0
+ #toAdd[4] = 1
+ #toAdd[5] = 0
+ #toAdd[6] = 0
- toAdd[7] = 1
- toAdd[17] = 1
+ #toAdd[7] = 1
+ #toAdd[17] = 1
+ toAdd = [0,2,4,7,17]
one_hot_bad_sub_out.append(toAdd)
else:
toAdd = []
@@ -665,6 +681,8 @@ def perform(curr):
toAdd[0] = 0
toAdd[1] = 1 # FIRST BIT (01) - INDICATE NO ERROR (1 because rest are 0 and so add up to 1)
one_hot_bad_sub_out.append(toAdd)
+
+ '''
#print "Morning"
#print len(allGood)
#print len(all_tokens)
@@ -704,11 +722,20 @@ def perform(curr):
#sizes = [len(one_hot_good), len(one_hot_bad_ins),len(one_hot_bad_del),len(one_hot_bad_sub)]
#minSize = int(min(float(siz) for siz in sizes)) # min of a generator
#return minSize
-
-
- toPass = [one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub, one_hot_good_out, one_hot_bad_ins_out, one_hot_bad_del_out, one_hot_bad_sub_out, passInsErrorInd]
-
+ #toPassGood = []
+ #print len(one_hot_good)
+ #for good in one_hot_good:
+ # ind = good.index(1.0)
+ # toPassGood.append(ind)
+ #print len(toPassGood)
+ #print one_hot_bad_sub_out
+ #print type(radha)
+
+ toPass = [one_hot_good, one_hot_bad_ins, one_hot_bad_del, one_hot_bad_sub]
+ #toPass = sparse.csr_matrix(toPassMatrix)
+ #print toPass
+ #print type(radha)
#cPickle.dump(one_hot_bad_ins, fileStore)
#cPickle.dump(one_hot_bad_del, fileStore)
@@ -738,16 +765,26 @@ def giveItems():
#print allData[3][8]
if __name__ == '__main__':
- all_data_to_pass = []
- for x in range(2001):
+ all_train_to_pass = []
+ for x in range(1000):
print x
if x != -1:
toP = perform(x)
- all_data_to_pass.append(toP)
- fileStore = open("all_pre_data.txt", 'w')
- cPickle.dump(all_data_to_pass, fileStore)
+ all_train_to_pass.append(toP)
+ fileStore = open("train_pre_data.txt", 'w')
+ cPickle.dump(all_train_to_pass, fileStore)
fileStore.close()
- giveItems()
+ all_val_to_pass = []
+ for x in range(1000):
+ print x
+ if x != -1:
+ toPV = perform(x+1001)
+ all_val_to_pass.append(toPV)
+ fileStoreV = open("val_pre_data.txt", 'w')
+ cPickle.dump(all_val_to_pass, fileStoreV)
+ fileStoreV.close()
+ print "FINISHED"
+ #giveItems()
sys.exit()
for x in range(10):
diff --git a/py_mutations_hub.pyc b/py_mutations_hub.pyc
index abeee6a..ad42145 100644
Binary files a/py_mutations_hub.pyc and b/py_mutations_hub.pyc differ
diff --git a/toCheck.py b/toCheck.py
index 359501b..aa81100 100644
--- a/toCheck.py
+++ b/toCheck.py
@@ -1,570 +1,124 @@
-# -*- coding: utf-8 -*-
-# Licensed to the StackStorm, Inc ('StackStorm') under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import mock
-
-from st2common.exceptions.param import ParamException
-from st2common.models.system.common import ResourceReference
-from st2common.models.db.liveaction import LiveActionDB
-from st2common.models.db.keyvalue import KeyValuePairDB
-from st2common.models.utils import action_param_utils
-from st2common.persistence.keyvalue import KeyValuePair
-from st2common.transport.publishers import PoolPublisher
-from st2common.util import date as date_utils
-from st2common.util import param as param_utils
-from st2tests import DbTestCase
-from st2tests.fixturesloader import FixturesLoader
-
-
-FIXTURES_PACK = 'generic'
-
-TEST_MODELS = {
- 'actions': ['action_4_action_context_param.yaml', 'action_system_default.yaml'],
- 'runners': ['testrunner1.yaml']
-}
-
-FIXTURES = FixturesLoader().load_models(fixtures_pack=FIXTURES_PACK,
- fixtures_dict=TEST_MODELS)
-
-
-@mock.patch.object(PoolPublisher, 'publish', mock.MagicMock())
-class ParamsUtilsTest(DbTestCase):
- action_db = FIXTURES['actions']['action_4_action_context_param.yaml']
- action_system_default_db = FIXTURES['actions']['action_system_default.yaml']
- runnertype_db = FIXTURES['runners']['testrunner1.yaml']
-
- def test_get_finalized_params(self):
- params = {
- 'actionstr': 'foo',
- 'some_key_that_aint_exist_in_action_or_runner': 'bar',
- 'runnerint': 555,
- 'runnerimmutable': 'failed_override',
- 'actionimmutable': 'failed_override'
- }
- liveaction_db = self._get_liveaction_model(params)
-
- runner_params, action_params = param_utils.get_finalized_params(
- ParamsUtilsTest.runnertype_db.runner_parameters, except
- ParamsUtilsTest.action_db.parameters,
- liveaction_db.parameters,
- liveaction_db.context)
-
- # Asserts for runner params.
- # Assert that default values for runner params are resolved.
- self.assertEqual(runner_params.get('runnerstr'), 'defaultfoo')
- # Assert that a runner param from action exec is picked up.
- self.assertEqual(runner_params.get('runnerint'), 555)
- # Assert that a runner param can be overridden by action param default.
- self.assertEqual(runner_params.get('runnerdummy'), 'actiondummy')
- # Assert that a runner param default can be overridden by 'falsey' action param default,
- # (timeout: 0 case).
- self.assertEqual(runner_params.get('runnerdefaultint'), 0)
- # Assert that an immutable param cannot be overridden by action param or execution param.
- self.assertEqual(runner_params.get('runnerimmutable'), 'runnerimmutable')
-
- # Asserts for action params.
- self.assertEqual(action_params.get('actionstr'), 'foo')
- # Assert that a param that is provided in action exec that isn't in action or runner params
- # isn't in resolved params.
- self.assertEqual(action_params.get('some_key_that_aint_exist_in_action_or_runner'), None)
- # Assert that an immutable param cannot be overridden by execution param.
- self.assertEqual(action_params.get('actionimmutable'), 'actionimmutable')
- # Assert that an action context param is set correctly.
- self.assertEqual(action_params.get('action_api_user'), 'noob')
- # Assert that none of runner params are present in action_params.
- for k in action_params:
- self.assertTrue(k not in runner_params, 'Param ' + k + ' is a runner param.')
-
- def test_get_finalized_params_system_values(self):
- KeyValuePair.add_or_update(KeyValuePairDB(name='actionstr', value='foo'))
- KeyValuePair.add_or_update(KeyValuePairDB(name='actionnumber', value='1.0'))
- params = {
- 'runnerint': 555
- }
- liveaction_db = self._get_liveaction_model(params)
-
- runner_params, action_params = param_utils.get_finalized_params(
- ParamsUtilsTest.runnertype_db.runner_parameters,
- ParamsUtilsTest.action_system_default_db.parameters,
- liveaction_db.parameters,
- liveaction_db.context)
-
- # Asserts for runner params.
- # Assert that default values for runner params are resolved.
- self.assertEqual(runner_params.get('runnerstr'), 'defaultfoo')
- # Assert that a runner param from action exec is picked up.
- self.assertEqual(runner_params.get('runnerint'), 555)
- # Assert that an immutable param cannot be overridden by action param or execution param.
- self.assertEqual(runner_params.get('runnerimmutable'), 'runnerimmutable')
-
- # Asserts for action params.
- self.assertEqual(action_params.get('actionstr'), 'foo')
- self.assertEqual(action_params.get('actionnumber'), 1.0)
-
- def test_get_finalized_params_action_immutable(self):
- params = {
- 'actionstr': 'foo',
- 'some_key_that_aint_exist_in_action_or_runner': 'bar',
- 'runnerint': 555,
- 'actionimmutable': 'failed_override'
- }
- liveaction_db = self._get_liveaction_model(params)
- action_context = {'api_user': None}
-
- runner_params, action_params = param_utils.get_finalized_params(
- ParamsUtilsTest.runnertype_db.runner_parameters,
- ParamsUtilsTest.action_db.parameters,
- liveaction_db.parameters,
- action_context)
-
- # Asserts for runner params.
- # Assert that default values for runner params are resolved.
- self.assertEqual(runner_params.get('runnerstr'), 'defaultfoo')
- # Assert that a runner param from action exec is picked up.
- self.assertEqual(runner_params.get('runnerint'), 555)
- # Assert that a runner param can be overridden by action param default.
- self.assertEqual(runner_params.get('runnerdummy'), 'actiondummy')
-
- # Asserts for action params.
- self.assertEqual(action_params.get('actionstr'), 'foo')
- # Assert that a param that is provided in action exec that isn't in action or runner params
- # isn't in resolved params.
- self.assertEqual(action_params.get('some_key_that_aint_exist_in_action_or_runner'), None)
-
- def test_get_finalized_params_empty(self):
- params = {}
- runner_param_info = {}
- action_param_info = {}
- action_context = {}
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.assertEqual(r_runner_params, params)
- self.assertEqual(r_action_params, params)
-
- def test_get_finalized_params_none(self):
- params = {
- 'r1': None,
- 'a1': None
- }
- runner_param_info = {'r1': {}}
- action_param_info = {'a1': {}}
- action_context = {'api_user': None}
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.assertEqual(r_runner_params, {'r1': None})
- self.assertEqual(r_action_params, {'a1': None})
-
- def test_get_finalized_params_no_cast(self):
- params = {
- 'r1': '{{r2}}',
- 'r2': 1,
- 'a1': True,
- 'a2': '{{r1}} {{a1}}',
- 'a3': '{{action_context.api_user}}'
- }
- runner_param_info = {'r1': {}, 'r2': {}}
- action_param_info = {'a1': {}, 'a2': {}, 'a3': {}}
- action_context = {'api_user': 'noob'}
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.assertEqual(r_runner_params, {'r1': u'1', 'r2': 1})
- self.assertEqual(r_action_params, {'a1': True, 'a2': u'1 True', 'a3': 'noob'})
-
- def test_get_finalized_params_with_cast(self):
- # Note : In this test runner_params.r1 has a string value. However per runner_param_info the
- # type is an integer. The expected type is considered and cast is performed accordingly.
- params = {
- 'r1': '{{r2}}',
- 'r2': 1,
- 'a1': True,
- 'a2': '{{a1}}',
- 'a3': '{{action_context.api_user}}'
- }
- runner_param_info = {'r1': {'type': 'integer'}, 'r2': {'type': 'integer'}}
- action_param_info = {'a1': {'type': 'boolean'}, 'a2': {'type': 'boolean'}, 'a3': {}}
- action_context = {'api_user': 'noob'}
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.assertEqual(r_runner_params, {'r1': 1, 'r2': 1})
- self.assertEqual(r_action_params, {'a1': True, 'a2': True, 'a3': 'noob'})
-
- def test_get_finalized_params_with_cast_overriden(self):
- params = {
- 'r1': '{{r2}}',
- 'r2': 1,
- 'a1': '{{r1}}',
- 'a2': '{{r1}}',
- 'a3': '{{r1}}'
- }
- runner_param_info = {'r1': {'type': 'integer'}, 'r2': {'type': 'integer'}}
- action_param_info = {'a1': {'type': 'boolean'}, 'a2': {'type': 'string'},
- 'a3': {'type': 'integer'}, 'r1': {'type': 'string'}}
- action_context = {'api_user': 'noob'}
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.assertEqual(r_runner_params, {'r1': 1, 'r2': 1})
- self.assertEqual(r_action_params, {'a1': 1, 'a2': u'1', 'a3': 1})
-
- def test_get_finalized_params_cross_talk_no_cast(self):
- params = {
- 'r1': '{{a1}}',
- 'r2': 1,
- 'a1': True,
- 'a2': '{{r1}} {{a1}}',
- 'a3': '{{action_context.api_user}}'
- }
- runner_param_info = {'r1': {}, 'r2': {}}
- action_param_info = {'a1': {}, 'a2': {}, 'a3': {}}
- action_context = {'api_user': 'noob'}
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.assertEqual(r_runner_params, {'r1': u'True', 'r2': 1})
- self.assertEqual(r_action_params, {'a1': True, 'a2': u'True True', 'a3': 'noob'})
-
- def test_get_finalized_params_cross_talk_with_cast(self):
- params = {
- 'r1': '{{a1}}',
- 'r2': 1,
- 'r3': 1,
- 'a1': True,
- 'a2': '{{r1}},{{a1}},{{a3}},{{r3}}',
- 'a3': '{{a1}}'
- }
- runner_param_info = {'r1': {'type': 'boolean'}, 'r2': {'type': 'integer'}, 'r3': {}}
- action_param_info = {'a1': {'type': 'boolean'}, 'a2': {'type': 'array'}, 'a3': {}}
- action_context = {}
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.assertEqual(r_runner_params, {'r1': True, 'r2': 1, 'r3': 1})
- self.assertEqual(r_action_params, {'a1': True, 'a2': (True, True, True, 1), 'a3': u'True'})
-
- def test_get_finalized_params_order(self):
- params = {
- 'r1': 'p1',
- 'r2': 'p2',
- 'r3': 'p3',
- 'a1': 'p4',
- 'a2': 'p5'
- }
- runner_param_info = {'r1': {}, 'r2': {'default': 'r2'}, 'r3': {'default': 'r3'}}
- action_param_info = {'a1': {}, 'a2': {'default': 'a2'}, 'r3': {'default': 'a3'}}
- action_context = {'api_user': 'noob'}
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.assertEqual(r_runner_params, {'r1': u'p1', 'r2': u'p2', 'r3': u'p3'})
- self.assertEqual(r_action_params, {'a1': u'p4', 'a2': u'p5'})
-
- params = {}
- runner_param_info = {'r1': {}, 'r2': {'default': 'r2'}, 'r3': {'default': 'r3'}}
- action_param_info = {'a1': {}, 'a2': {'default': 'a2'}, 'r3': {'default': 'a3'}}
- action_context = {'api_user': 'noob'}
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.assertEqual(r_runner_params, {'r1': None, 'r2': u'r2', 'r3': u'a3'})
- self.assertEqual(r_action_params, {'a1': None, 'a2': u'a2'})
-
- params = {}
- runner_param_info = {'r1': {}, 'r2': {'default': 'r2'}, 'r3': {}}
- action_param_info = {'r1': {}, 'r2': {}, 'r3': {'default': 'a3'}}
- action_context = {'api_user': 'noob'}
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.assertEqual(r_runner_params, {'r1': None, 'r2': u'r2', 'r3': u'a3'})
-
- def test_get_finalized_params_non_existent_template_key_in_action_context(self):
- params = {
- 'r1': 'foo',
- 'r2': 2,
- 'a1': 'i love tests',
- 'a2': '{{action_context.lorem_ipsum}}'
- }
- runner_param_info = {'r1': {'type': 'string'}, 'r2': {'type': 'integer'}}
- action_param_info = {'a1': {'type': 'string'}, 'a2': {'type': 'string'}}
- action_context = {'api_user': 'noob', 'source_channel': 'reddit'}
- try:
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.fail('This should have thrown because we are trying to deref a key in ' +
- 'action context that ain\'t exist.')
- except ParamException as e:
- error_msg = 'Failed to render parameter "a2": \'dict object\' ' + \
- 'has no attribute \'lorem_ipsum\''
- self.assertTrue(error_msg in e.message)
- pass
-
- def test_unicode_value_casting(self):
- rendered = {'a1': 'unicode1 ٩(̾●̮̮̃̾•̃̾)۶ unicode2'}
- parameter_schemas = {'a1': {'type': 'string'}}
-
- result = param_utils._cast_params(rendered=rendered,
- parameter_schemas=parameter_schemas)
- expected = {
- 'a1': (u'unicode1 \xd9\xa9(\xcc\xbe\xe2\x97\x8f\xcc\xae\xcc\xae\xcc'
- u'\x83\xcc\xbe\xe2\x80\xa2\xcc\x83\xcc\xbe)\xdb\xb6 unicode2')
- }
- self.assertEqual(result, expected)
-
- def test_get_finalized_params_with_casting_unicode_values(self):
- params = {'a1': 'unicode1 ٩(̾●̮̮̃̾•̃̾)۶ unicode2'}
-
- runner_param_info = {}
- action_param_info = {'a1': {'type': 'string'}}
-
- action_context = {}
-
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
-
- expected_action_params = {
- 'a1': (u'unicode1 \xd9\xa9(\xcc\xbe\xe2\x97\x8f\xcc\xae\xcc\xae\xcc'
- u'\x83\xcc\xbe\xe2\x80\xa2\xcc\x83\xcc\xbe)\xdb\xb6 unicode2')
- }
- self.assertEqual(r_runner_params, {})
- self.assertEqual(r_action_params, expected_action_params)
-
- def test_get_finalized_params_with_dict(self):
- # Note : In this test runner_params.r1 has a string value. However per runner_param_info the
- # type is an integer. The expected type is considered and cast is performed accordingly.
- params = {
- 'r1': '{{r2}}',
- 'r2': {'r2.1': 1},
- 'a1': True,
- 'a2': '{{a1}}',
- 'a3': {
- 'test': '{{a1}}',
- 'test1': '{{a4}}',
- 'test2': '{{a5}}',
- },
- 'a4': 3,
- 'a5': ['1', '{{a1}}']
- }
- runner_param_info = {'r1': {'type': 'object'}, 'r2': {'type': 'object'}}
- action_param_info = {
- 'a1': {
- 'type': 'boolean',
- },
- 'a2': {
- 'type': 'boolean',
- },
- 'a3': {
- 'type': 'object',
- },
- 'a4': {
- 'type': 'integer',
- },
- 'a5': {
- 'type': 'array',
- },
- }
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, {})
- self.assertEqual(
- r_runner_params, {'r1': {'r2.1': 1}, 'r2': {'r2.1': 1}})
- self.assertEqual(
- r_action_params,
- {
- 'a1': True,
- 'a2': True,
- 'a3': {
- 'test': True,
- 'test1': 3,
- 'test2': [
- '1',
- True
- ],
- },
- 'a4': 3,
- 'a5': [
- '1',
- True
- ],
- }
- )
-
- def test_get_finalized_params_with_list(self):
- # Note : In this test runner_params.r1 has a string value. However per runner_param_info the
- # type is an integer. The expected type is considered and cast is performed accordingly.
- self.maxDiff = None
- params = {
- 'r1': '{{r2}}',
- 'r2': ['1', '2'],
- 'a1': True,
- 'a2': 'Test',
- 'a3': 'Test2',
- 'a4': '{{a1}}',
- 'a5': ['{{a2}}', '{{a3}}'],
- 'a6': [
- ['{{r2}}', '{{a2}}'],
- ['{{a3}}', '{{a1}}'],
- [
- '{{a7}}',
- 'This should be rendered as a string {{a1}}',
- '{{a1}} This, too, should be rendered as a string {{a1}}',
- ]
- ],
- 'a7': 5,
- }
- runner_param_info = {'r1': {'type': 'array'}, 'r2': {'type': 'array'}}
- action_param_info = {
- 'a1': {'type': 'boolean'},
- 'a2': {'type': 'string'},
- 'a3': {'type': 'string'},
- 'a4': {'type': 'boolean'},
- 'a5': {'type': 'array'},
- 'a6': {'type': 'array'},
- 'a7': {'type': 'integer'},
- }
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, {})
- self.assertEqual(r_runner_params, {'r1': ['1', '2'], 'r2': ['1', '2']})
- self.assertEqual(
- r_action_params,
- {
- 'a1': True,
- 'a2': 'Test',
- 'a3': 'Test2',
- 'a4': True,
- 'a5': ['Test', 'Test2'],
- 'a6': [
- [['1', '2'], 'Test'],
- ['Test2', True],
- [
- 5,
- u'This should be rendered as a string True',
- u'True This, too, should be rendered as a string True'
- ]
- ],
- 'a7': 5,
- }
- )
-
- def test_get_finalized_params_with_cyclic_dependency(self):
- params = {'r1': '{{r2}}', 'r2': '{{r1}}'}
- runner_param_info = {'r1': {}, 'r2': {}}
- action_param_info = {}
- test_pass = True
- try:
- param_utils.get_finalized_params(runner_param_info, action_param_info, params, {})
- test_pass = False
- except ParamException as e:
- test_pass = e.message.find('Cyclic') == 0
- self.assertTrue(test_pass)
-
- def test_get_finalized_params_with_missing_dependency(self):
- params = {'r1': '{{r3}}', 'r2': '{{r3}}'}
- runner_param_info = {'r1': {}, 'r2': {}}
- action_param_info = {}
- test_pass = True
- try:
- param_utils.get_finalized_params(runner_param_info, action_param_info, params, {})
- test_pass = False
- except ParamException as e:
- test_pass = e.message.find('Dependecy') == 0
- self.assertTrue(test_pass)
-
- params = {}
- runner_param_info = {'r1': {'default': '{{r3}}'}, 'r2': {'default': '{{r3}}'}}
- action_param_info = {}
- test_pass = True
- try:
- param_utils.get_finalized_params(runner_param_info, action_param_info, params, {})
- test_pass = False
- except ParamException as e:
- test_pass = e.message.find('Dependecy') == 0
- self.assertTrue(test_pass)
-
- def test_get_finalized_params_no_double_rendering(self):
- params = {
- 'r1': '{{ action_context.h1 }}{{ action_context.h2 }}'
- }
- runner_param_info = {'r1': {}}
- action_param_info = {}
- action_context = {
- 'h1': '{',
- 'h2': '{ missing }}'
- }
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
- self.assertEqual(r_runner_params, {'r1': '{{ missing }}'})
- self.assertEqual(r_action_params, {})
-
- def test_get_finalized_params_jinja_filters(self):
- params = {'cmd': 'echo {{"1.6.0" | version_bump_minor}}'}
- runner_param_info = {'r1': {}}
- action_param_info = {'cmd': {}}
- action_context = {}
-
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
-
- self.assertEqual(r_action_params['cmd'], "echo 1.7.0")
-
- def test_get_finalized_params_param_rendering_failure(self):
- params = {'cmd': '{{a2.foo}}', 'a2': 'test'}
- action_param_info = {'cmd': {}, 'a2': {}}
-
- expected_msg = 'Failed to render parameter "cmd": .*'
- self.assertRaisesRegexp(ParamException,
- expected_msg,
- param_utils.get_finalized_params,
- runnertype_parameter_info={},
- action_parameter_info=action_param_info,
- liveaction_parameters=params,
- action_context={})
-
- def test_get_finalized_param_object_contains_template_notation_in_the_value(self):
- runner_param_info = {'r1': {}}
- action_param_info = {
- 'params': {
- 'type': 'object',
- 'default': {
- 'host': '{{host}}',
- 'port': '{{port}}',
- 'path': '/bar'}
- }
- }
- params = {
- 'host': 'lolcathost',
- 'port': 5555
- }
- action_context = {}
-
- r_runner_params, r_action_params = param_utils.get_finalized_params(
- runner_param_info, action_param_info, params, action_context)
-
- expected_params = {
- 'host': 'lolcathost',
- 'port': 5555,
- 'path': '/bar'
- }
- self.assertEqual(r_action_params['params'], expected_params)
-
- def test_cast_param_referenced_action_doesnt_exist(self):
- # Make sure the function throws if the action doesnt exist
- expected_msg = 'Action with ref "foo.doesntexist" doesn\'t exist'
- self.assertRaisesRegexp(ValueError, expected_msg, action_param_utils.cast_params,
- action_ref='foo.doesntexist', params={})
-
- def _get_liveaction_model(self, params):
- status = 'initializing'
- start_timestamp = date_utils.get_datetime_utc_now()
- action_ref = ResourceReference(name=ParamsUtilsTest.action_db.name,
- pack=ParamsUtilsTest.action_db.pack).ref
- liveaction_db = LiveActionDB(status=status, start_timestamp=start_timestamp,
- action=action_ref, parameters=params)
- liveaction_db.context = {'source_channel': 'awesome', 'api_user': 'noob'}
-
- return liveaction_db
+from __future__ import absolute_import
+
+from django.db import transaction, IntegrityError
+from django.test import TestCase
+
+from .models import (Place, Restaurant, Waiter, ManualPrimaryKey, RelatedModel,
+ MultiModel)
+
+class OneToOneTests(TestCase):
+
+ def setUp(self):
+ self.p1 = Place(name='Demon Dogs', address='944 W. Fullerton')
+ self.p1.save()
+ self.p2 = Place(name='Ace Hardware', address='1013 N. Ashland')
+ self.p2.save()
+ self.r = Restaurant(place=self.p1, serves_hot_dogs=True, serves_pizza=False)
+ self.r.save()
+
+ def test_getter(self):
+ # A Restaurant can access its place.
+ self.assertEqual(repr(self.r.place), '')
+ # A Place can access its restaurant, if available.
+ self.assertEqual(repr(self.p1.restaurant), '')
+ # p2 doesn't have an associated restaurant.
+ with self.assertRaisesMessage(Restaurant.DoesNotExist, 'Place has no restaurant'):
+ self.p2.restaurant
+
+ def test_setter(self):
+ # Set the place using assignment notation. Because place is the primary
+ # key on Restaurant, the save will create a new restaurant
+ self.r.place = self.p2
+ self.r.save()
+ self.assertEqual(repr(self.p2.restaurant), '')
+ self.assertEqual(repr(self.r.place), '')
+ self.assertEqual(self.p2.pk, self.r.pk)
+ # Set the place back again, using assignment in the reverse direction.
+ self.p1.restaurant = self.r
+ self.assertEqual(repr(self.p1.restaurant), '')
+ r = Restaurant.objects.get(pk=self.p1.id)
+ self.assertEqual(repr(r.place), '')
+
+ def test_manager_all(self):
+ # Restaurant.objects.all() just returns the Restaurants, not the Places.
+ self.assertQuerysetEqual(Restaurant.objects.all(), [
+ '',
+ ])
+ # Place.objects.all() returns all Places, regardless of whether they
+ # have Restaurants.
+ self.assertQuerysetEqual(Place.objects.order_by('name'), [
+ '',
+ '',
+ ])
+
+ def test_manager_get(self):
+ def assert_get_restaurant(**params):
+ self.assertEqual(repr(Restaurant.objects.get(**params)),
+ '')
+ assert_get_restaurant(place__id__exact=self.p1.pk)
+ assert_get_restaurant(place__id=self.p1.pk)
+ assert_get_restaurant(place__exact=self.p1.pk)
+ assert_get_restaurant(place__exact=self.p1)
+ assert_get_restaurant(place=self.p1.pk)
+ assert_get_restaurant(place=self.p1)
+ assert_get_restaurant(pk=self.p1.pk)
+ assert_get_restaurant(place__pk__exact=self.p1.pk)
+ assert_get_restaurant(place__pk=self.p1.pk)
+ assert_get_restaurant(place__name__startswith="Demon")
+
+ def assert_get_place(**params):
+ self.assertEqual(repr(Place.objects.get(**params)),
+ '')
+ assert_get_place(restaurant__place__exact=self.p1.pk)
+ assert_get_place(restaurant__place__exact=self.p1)
+ assert_get_place(restaurant__place__pk=self.p1.pk)
+ assert_get_place(restaurant__exact=self.p1.pk)
+ assert_get_place(restaurant__exact=self.r)
+ assert_get_place(restaurant__pk=self.p1.pk)
+ assert_get_place(restaurant=self.p1.pk)
+ assert_get_place(restaurant=self.r)
+ assert_get_place(id__exact=self.p1.pk)
+ assert_get_place(pk=self.p1.pk)
+
+ def test_foreign_key(self):
+ # Add a Waiter to the Restaurant.
+ w = self.r.waiter_set.create(name='Joe')
+ w.save()
+ self.assertEqual(repr(w), '')
+ # Query the waiters
+ def assert_filter_waiters(**params):
+ self.assertQuerysetEqual(Waiter.objects.filter(**params), [
+ ''
+ ])
+ assert_filter_waiters(restaurant__place__exact=self.p1.pk)
+ assert_filter_waiters(restaurant__place__exact=self.p1)
+ assert_filter_waiters(restaurant__place__pk=self.p1.pk)
+ assert_filter_waiters(restaurant__exact=self.p1.pk)
+ assert_filter_waiters(restaurant__exact=self.p1)
+ assert_filter_waiters(restaurant__pk=self.p1.pk)
+ assert_filter_waiters(restaurant=self.p1.pk)
+ assert_filter_waiters(restaurant=self.r)
+ assert_filter_waiters(id__exact=self.p1.pk)
+ assert_filter_waiters(pk=self.p1.pk)
+ # Delete the restaurant; the waiter should also be removed
+ r = Restaurant.objects.get(pk=self.p1.pk)
+ r.delete()
+ self.assertEqual(Waiter.objects.count(), 0)
+
+ def test_multiple_o2o(self):
+ # One-to-one fields still work if you create your own primary key
+ o1 = ManualPrimaryKey(primary_key="abc123", name="primary")
+ o1.save()
+ o2 = RelatedModel(link=o1, name="secondary")
+ o2.save()
+
+ # You can have multiple one-to-one fields on a model, too.
+ x1 = MultiModel(link1=self.p1, link2=o1, name="x1")
+ x1.save()
+ self.assertEqual(repr(o1.multimodel), '')
+ # This will fail because each one-to-one field must be unique (and
+ # link2=o1 was used for x1, above).
+ mm = MultiModel(link1=self.p2, link2=o1, name="x1")
+ with self.assertRaises(IntegrityError):
+ with transaction.atomic()-=
+ mm.save()