Skip to content

Commit

Permalink
Tracking PR for release-sprint-10 (#151)
Browse files Browse the repository at this point in the history 
Implemented JIRA cards:
ZEVA 132
ZEVA 133
ZEVA 134
ZEVA 135
ZEVA 136
ZEVA 137
ZEVA 139
  • Loading branch information
@kuanfandevops
kuanfandevops authored May 5, 2020
1 parent 4f241a1 commit 2b7e6d4
Show file tree
Hide file tree
Showing 103 changed files with 6,507 additions and 524 deletions.
52 changes: 26 additions & 26 deletions .pipeline/lib/config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,42 +6,42 @@ const name = 'zeva'

const phases = {

build: {namespace:'tbiwaq-tools' , transient:true, name: `${name}`, phase: 'build' ,
changeId:changeId, suffix: `-build-${changeId}` , instance: `${name}-build-${changeId}` ,
build: {namespace:'tbiwaq-tools' , transient:true, name: `${name}`, phase: 'build',
changeId:changeId, suffix: `-build-${changeId}` , instance: `${name}-build-${changeId}`,
version:`${version}-${changeId}`, tag:`build-${version}-${changeId}`},

dev: {namespace:'tbiwaq-dev' , transient:true, name: `${name}`, ssoSuffix:'-dev',
ssoName:'sso-dev', phase: 'dev' , changeId:changeId, suffix: `-dev-${changeId}` ,
dev: {namespace:'tbiwaq-dev', transient:true, name: `${name}`, ssoSuffix:'-dev',
ssoName:'sso-dev', phase: 'dev' , changeId:changeId, suffix: `-dev-${changeId}`,
instance: `${name}-dev-${changeId}` , version:`${version}-${changeId}`, tag:`dev-${version}-${changeId}`,
host: `zeva-dev-${changeId}.pathfinder.gov.bc.ca`, minioPvcSize: '1Gi', djangoDebug: 'True',
frontendCpuRequest: '50m', frontendCpuLimit: '300m', frontendMemoryRequest: '300M', frontendMemoryLimit: '500M',
backendCpuRequest: '300m', backendCpuLimit: '600m', backendMemoryRequest: '1G', backendMemoryLimit: '2G', backendHealthCheckDelay: 10,
backendCpuRequest: '300m', backendCpuLimit: '600m', backendMemoryRequest: '1G', backendMemoryLimit: '2G', backendHealthCheckDelay: 140,
minioCpuRequest: '50m', minioCpuLimit: '200m', minioMemoryRequest: '500M', minioMemoryLimit: '700M',
schemaspyCpuRequest: '50m', schemaspyCpuLimit: '200m', schemaspyMemoryRequest: '150M', schemaspyMemoryLimit: '300M', schemaspyHealthCheckDelay: 30,
rabbitmqCpuRequest: '250m', rabbitmqCpuLimit: '400m', rabbitmqMemoryRequest: '500M', rabbitmqMemoryLimit: '700M', rabbitmqPvcSize: '1G', rabbitmqPodManagementPolicy: 'Parallel', rabbitmqReplica: 2, rabbitmqPostStartSleep: 120,
patroniCpuRequest: '200m', patroniCpuLimit: '400m', patroniMemoryRequest: '200M', patroniMemoryLimit: '400M', patroniPvcSize: '1G', patroniPodManagementPolicy: 'OrderedReady', patroniReplica: 2},
schemaspyCpuRequest: '50m', schemaspyCpuLimit: '200m', schemaspyMemoryRequest: '150M', schemaspyMemoryLimit: '300M', schemaspyHealthCheckDelay: 160,
rabbitmqCpuRequest: '250m', rabbitmqCpuLimit: '700m', rabbitmqMemoryRequest: '500M', rabbitmqMemoryLimit: '1G', rabbitmqPvcSize: '1G', rabbitmqReplica: 1, rabbitmqPostStartSleep: 120,
patroniCpuRequest: '200m', patroniCpuLimit: '400m', patroniMemoryRequest: '200M', patroniMemoryLimit: '400M', patroniPvcSize: '1G', patroniReplica: 1},

test: {namespace:'tbiwaq-test' , name: `${name}`, ssoSuffix:'-test',
ssoName:'sso-test', phase: 'test' , changeId:changeId, suffix: `-test` ,
instance: `${name}-test` , version:`${version}`, tag:`test-${version}`,
test: {namespace:'tbiwaq-test', name: `${name}`, ssoSuffix:'-test',
ssoName:'sso-test', phase: 'test' , changeId:changeId, suffix: `-test`,
instance: `${name}-test`, version:`${version}`, tag:`test-${version}`,
host: 'zeva-test.pathfinder.gov.bc.ca', minioPvcSize: '5Gi', djangoDebug: 'False',
frontendCpuRequest: '50m', frontendCpuLimit: '300m', frontendMemoryRequest: '300M', frontendMemoryLimit: '500M',
backendCpuRequest: '100m', backendCpuLimit: '400m', backendMemoryRequest: '500M', backendMemoryLimit: '700M', backendHealthCheckDelay: 10,
frontendCpuRequest: '50m', frontendCpuLimit: '300m', frontendMemoryRequest: '300M', frontendMemoryLimit: '500M', frontendMinReplicas: 2, frontendMaxReplicas: 5,
backendCpuRequest: '100m', backendCpuLimit: '400m', backendMemoryRequest: '500M', backendMemoryLimit: '700M', backendHealthCheckDelay: 140, backendMinReplicas: 2, backendMaxReplicas: 5,
minioCpuRequest: '50m', minioCpuLimit: '200m', minioMemoryRequest: '500M', minioMemoryLimit: '700M',
schemaspyCpuRequest: '20m', schemaspyCpuLimit: '200m', schemaspyMemoryRequest: '150M', schemaspyMemoryLimit: '300M', schemaspyHealthCheckDelay: 30,
rabbitmqCpuRequest: '250m', rabbitmqCpuLimit: '400m', rabbitmqMemoryRequest: '500M', rabbitmqMemoryLimit: '700', rabbitmqPvcSize: '1G', rabbitmqPodManagementPolicy: 'OrderedReady', rabbitmqReplica: 3, rabbitmqPostStartSleep: 120,
patroniCpuRequest: '200m', patroniCpuLimit: '400m', patroniMemoryRequest: '200M', patroniMemoryLimit: '400M', patroniPvcSize: '1G', patroniPodManagementPolicy: 'OrderedReady', patroniReplica: 3},

prod: {namespace:'tbiwaq-prod' , name: `${name}`, ssoSuffix:'',
ssoName:'sso', phase: 'prod' , changeId:changeId, suffix: `-prod` ,
instance: `${name}-prod` , version:`${version}`, tag:`prod-${version}`,
host: 'zeva-prod.pathfinder.gov.bc.ca', minioPvcSize: '10Gi', djangoDebug: 'False',
frontendCpuRequest: '50m', frontendCpuLimit: '100m', frontendMemoryRequest: '500M', frontendMemoryLimit: '700M',
backendCpuRequest: '100m', backendCpuLimit: '500m', backendMemoryRequest: '1G', backendMemoryLimit: '2G', backendHealthCheckDelay: 90,
schemaspyCpuRequest: '20m', schemaspyCpuLimit: '200m', schemaspyMemoryRequest: '150M', schemaspyMemoryLimit: '300M', schemaspyHealthCheckDelay: 160,
rabbitmqCpuRequest: '250m', rabbitmqCpuLimit: '700m', rabbitmqMemoryRequest: '500M', rabbitmqMemoryLimit: '700', rabbitmqPvcSize: '1G', rabbitmqReplica: 2, rabbitmqPostStartSleep: 120,
patroniCpuRequest: '200m', patroniCpuLimit: '400m', patroniMemoryRequest: '200M', patroniMemoryLimit: '400M', patroniPvcSize: '1G', patroniReplica: 2},

prod: {namespace:'tbiwaq-prod', name: `${name}`, ssoSuffix:'',
ssoName:'sso', phase: 'prod' , changeId:changeId, suffix: `-prod`,
instance: `${name}-prod`, version:`${version}`, tag:`prod-${version}`,
host: 'zeroemissionvehicles.pathfinder.gov.bc.ca', minioPvcSize: '10Gi', djangoDebug: 'False',
frontendCpuRequest: '100m', frontendCpuLimit: '300m', frontendMemoryRequest: '500M', frontendMemoryLimit: '1G', frontendMinReplicas: 2, frontendMaxReplicas: 5,
backendCpuRequest: '200m', backendCpuLimit: '500m', backendMemoryRequest: '1G', backendMemoryLimit: '2G', backendHealthCheckDelay: 140, backendMinReplicas: 2, backendMaxReplicas: 5,
minioCpuRequest: '100m', minioCpuLimit: '300m', minioMemoryRequest: '500M', minioMemoryLimit: '700M',
schemaspyCpuRequest: '50m', schemaspyCpuLimit: '500m', schemaspyMemoryRequest: '250M', schemaspyMemoryLimit: '1500M', schemaspyHealthCheckDelay: 90,
rabbitmqCpuRequest: '250m', rabbitmqCpuLimit: '600m', rabbitmqMemoryRequest: '512M', rabbitmqMemoryLimit: '1G', rabbitmqPvcSize: '2G', rabbitmqPodManagementPolicy: 'OrderedReady', rabbitmqReplica: 3, rabbitmqPostStartSleep: 120,
patroniCpuRequest: '250m', patroniCpuLimit: '600m', patroniMemoryRequest: '1G', patroniMemoryLimit: '2G', patroniPvcSize: '5G', patroniPodManagementPolicy: 'OrderedReady', patroniReplica: 3},
schemaspyCpuRequest: '50m', schemaspyCpuLimit: '400m', schemaspyMemoryRequest: '150M', schemaspyMemoryLimit: '300M', schemaspyHealthCheckDelay: 160,
rabbitmqCpuRequest: '250m', rabbitmqCpuLimit: '700m', rabbitmqMemoryRequest: '500M', rabbitmqMemoryLimit: '1G', rabbitmqPvcSize: '5G', rabbitmqReplica: 2, rabbitmqPostStartSleep: 120,
patroniCpuRequest: '250m', patroniCpuLimit: '600m', patroniMemoryRequest: '1G', patroniMemoryLimit: '2G', patroniPvcSize: '5G', patroniReplica: 2},

};

Expand Down
44 changes: 36 additions & 8 deletions .pipeline/lib/deploy.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,30 +68,36 @@ module.exports = settings => {
'IMAGE_REGISTRY': 'docker-registry.default.svc:5000',
'IMAGE_STREAM_NAMESPACE': phases[phase].namespace,
'IMAGE_STREAM_TAG': 'patroni:v10-stable',
'REPLICA': phases[phase].patroniReplica,
'POD_MANAGEMENT_POLICY': phases[phase].patroniPodManagementPolicy
'REPLICA': phases[phase].patroniReplica
}
}))

//deploy rabbitmq, use docker image directly
//TODO: tag docker image to local
//POST_START_SLEEP is harded coded in the rabbitmq template, replacement was not successful
objects = objects.concat(oc.processDeploymentTemplate(`${templatesLocalBaseUrl}/templates/rabbitmq/rabbitmq-cluster-dc.yaml`, {
'param': {
'NAME': phases[phase].name,
'ENV_NAME': phases[phase].phase,
'SUFFIX': phases[phase].suffix,
'NAMESPACE': phases[phase].namespace,
'CLUSTER_NAME': 'rabbitmq-cluster',
'ISTAG': 'rabbitmq:3.8.3-management',
'ISTAG': `docker-registry.default.svc:5000/${phases[phase].namespace}/rabbitmq:3.8.3-management`,
'SERVICE_ACCOUNT': 'rabbitmq-discovery',
'VOLUME_SIZE': phases[phase].rabbitmqPvcSize,
'CPU_REQUEST': phases[phase].rabbitmqCpuRequest,
'CPU_LIMIT': phases[phase].rabbitmqCpuLimit,
'MEMORY_REQUEST': phases[phase].rabbitmqMemoryRequest,
'MEMORY_LIMIT': phases[phase].rabbitmqMemoryLimit,
'REPLICA': phases[phase].rabbitmqReplica,
'POST_START_SLEEP': phases[phase].rabbitmqPostStartSleep,
'POD_MANAGEMENT_POLICY': phases[phase].rabbitmqPodManagementPolicy
'POST_START_SLEEP': phases[phase].rabbitmqPostStartSleep
}
}))

// deploy frontend configmap
objects = objects.concat(oc.processDeploymentTemplate(`${templatesLocalBaseUrl}/templates/frontend/frontend-configmap.yaml`, {
'param': {
'NAME': phases[phase].name,
'SUFFIX': phases[phase].suffix
}
}))

Expand All @@ -102,7 +108,7 @@ module.exports = settings => {
'SUFFIX': phases[phase].suffix,
'VERSION': phases[phase].tag,
'ENV_NAME': phases[phase].phase,
'DASH_ENV_NAME': phases[phase].ssoSuffix,
'HOST_NAME': phases[phase].host,
'CPU_REQUEST': phases[phase].frontendCpuRequest,
'CPU_LIMIT': phases[phase].frontendCpuLimit,
'MEMORY_REQUEST': phases[phase].frontendMemoryRequest,
Expand All @@ -117,6 +123,8 @@ module.exports = settings => {
'SUFFIX': phases[phase].suffix,
'VERSION': phases[phase].tag,
'ENV_NAME': phases[phase].phase,
'HOST_NAME': phases[phase].host,
'RABBITMQ_CLUSTER_NAME': 'rabbitmq-cluster',
'CPU_REQUEST': phases[phase].backendCpuRequest,
'CPU_LIMIT': phases[phase].backendCpuLimit,
'MEMORY_REQUEST': phases[phase].backendMemoryRequest,
Expand All @@ -137,7 +145,27 @@ module.exports = settings => {
'HEALTH_CHECK_DELAY': phases[phase].schemaspyHealthCheckDelay
}
}))


//add autoacaler
if(phase === 'test' || phase === 'prod') {
objects = objects.concat(oc.processDeploymentTemplate(`${templatesLocalBaseUrl}/templates/frontend/frontend-autoscaler.yaml`, {
'param': {
'NAME': phases[phase].name,
'SUFFIX': phases[phase].suffix,
'MIN_REPLICAS': phases[phase].frontendMinReplicas,
'MAX_REPLICAS': phases[phase].frontendMaxReplicas
}
}))
objects = objects.concat(oc.processDeploymentTemplate(`${templatesLocalBaseUrl}/templates/backend/backend-autoscaler.yaml`, {
'param': {
'NAME': phases[phase].name,
'SUFFIX': phases[phase].suffix,
'MIN_REPLICAS': phases[phase].backendMinReplicas,
'MAX_REPLICAS': phases[phase].backendMaxReplicas
}
}))
}

oc.applyRecommendedLabels(
objects,
phases[phase].name,
Expand Down
57 changes: 27 additions & 30 deletions backend/api/keycloak_authentication.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,58 +121,55 @@ def authenticate(self, request):
)

user_found_via_email = None
user = None

if 'user_id' not in user_token:
# try email
if 'email' in user_token:
creation_request = UserCreationRequest.objects.filter(
user_profile = UserProfile.objects.filter(
keycloak_email__iexact=user_token['email']
)

if not creation_request.exists():
if not user_profile.exists():
raise exceptions.AuthenticationFailed(
"User does not exist.")

if creation_request.count() > 1:
if user_profile.count() > 1:
_, preferred_username = user_token[
'preferred_username'].split('\\')

creation_request = creation_request.filter(
external_username__iexact=preferred_username
user_profile = user_profile.filter(
username__iexact=preferred_username
)

user_creation_request = creation_request.first()
user = user_profile.first()

if not user_creation_request.is_mapped:
map_user(user_token['sub'],
user_creation_request.user.username)

user_creation_request.is_mapped = True
user_creation_request.save()

user_found_via_email = user_creation_request.user.username
map_user(
user_token.get('sub'),
user.username
)
else:
raise exceptions.AuthenticationFailed(
'user_id or email is required in jwt payload')

username = user_token['user_id'] \
if 'user_id' in user_token else user_found_via_email

try:
user = UserProfile.objects.get_by_natural_key(username)

if not user.is_active:
else:
try:
user = UserProfile.objects.get_by_natural_key(
user_token.get('user_id')
)
except UserProfile.DoesNotExist:
raise exceptions.AuthenticationFailed(
'user_id "{}" does not exist'.format(username))

if 'realm_access' in user_token:
if 'roles' in user_token['realm_access']:
for role in user_token['realm_access']['roles']:
if role not in FILTERED_ROLES:
user.roles.append(role)
if not user.is_active:
raise exceptions.PermissionDenied(
'Your account is currently inactive. Please contact your '
'administrator to re-activate your account.'
)

except UserProfile.DoesNotExist:
raise exceptions.AuthenticationFailed(
'user_id "{}" does not exist'.format(username))
if 'realm_access' in user_token:
if 'roles' in user_token['realm_access']:
for role in user_token['realm_access']['roles']:
if role not in FILTERED_ROLES:
user.roles.append(role)

return user, None
28 changes: 28 additions & 0 deletions backend/api/migrations/0036_auto_20200429_1327.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Generated by Django 3.0.3 on 2020-04-29 20:27

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('api', '0035_auto_20200416_0811'),
]

operations = [
migrations.AddField(
model_name='userprofile',
name='keycloak_email',
field=models.EmailField(blank=True, max_length=254, null=True),
),
migrations.AddField(
model_name='userprofile',
name='title',
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AlterField(
model_name='userprofile',
name='username',
field=models.CharField(max_length=130, unique=True),
),
]
8 changes: 7 additions & 1 deletion backend/api/models/user_profile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,17 @@ class UserProfile(Auditable):
max_length=100,
null=True,
db_comment="Professional Title"
),
)
username = models.CharField(
max_length=130,
unique=True,
db_comment="Username that we can connect the user to Keycloak."
)
keycloak_email = models.EmailField(
blank=True,
null=True,
db_comment="BCEID/IDIR Email Address"
)

# Unpersisted (supplied by Keycloak authentication filter)
roles = []
Expand Down
42 changes: 40 additions & 2 deletions backend/api/serializers/organization.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

from api.models.organization import Organization
from api.serializers.organization_address import OrganizationAddressSerializer

from api.models.organization_address import OrganizationAddress
from datetime import date

class OrganizationSerializer(serializers.ModelSerializer):
"""
Expand All @@ -28,7 +29,7 @@ class Meta:
model = Organization
fields = (
'id', 'name', 'organization_address', 'create_timestamp',
'balance'
'balance', 'is_active', 'short_name'
)


Expand All @@ -51,3 +52,40 @@ class Meta:
'id', 'name', 'organization_address', 'users', 'create_timestamp',
'balance'
)


class OrganizationSaveSerializer(serializers.ModelSerializer):
"""
Serializer for saving/editing the Supplier
Loads most of the fields and the balance for the Supplier
"""
organization_address = OrganizationAddressSerializer(allow_null=True)

def update(self, obj, validated_data):
addr = validated_data.pop('organization_address')
short_name = validated_data.pop('short_name')
is_active = validated_data.pop('is_active')
name = validated_data.pop('name')
obj.short_name = short_name
obj.is_active = is_active
obj.name = name
obj.save()
organization_address = obj.organization_address
if addr:
if organization_address:
organization_address.expiration_date = date.today()
organization_address.save()

OrganizationAddress.objects.create(
effective_date=date.today(),
organization=obj,
**addr
)
return obj

class Meta:
model = Organization
fields = (
'id', 'name', 'organization_address', 'create_timestamp',
'balance', 'is_active', 'short_name'
)
Loading

0 comments on commit 2b7e6d4

Please sign in to comment.