From cf3df4706f4f065dd93142cc0a5313c0cc2de4d4 Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Thu, 29 Oct 2020 19:21:00 -0300 Subject: [PATCH 01/12] Adiciona helper para mascara dos campos das models --- helpers.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/helpers.py b/helpers.py index ef40726..18e79b2 100644 --- a/helpers.py +++ b/helpers.py @@ -1,10 +1,11 @@ from flask import request, redirect, url_for, flash, session -from functools import wraps +from functools import partial, wraps from flask.globals import g from app import db -from models import Customers + +import re def login_required(view): @@ -30,7 +31,7 @@ def wrapper(*args, **kwargs): def commit_on_finish(f): """ - Adiciona os objetos atraves da sintaxe 'yield [objeto]' e faz commit. + Adiciona aos objetos atraves da sintaxe 'yield [objeto]' e faz commit. """ @wraps(f) def wrapper(self, *args, **kwargs): @@ -63,3 +64,18 @@ def wrapper(self, *args, **kwargs): kwargs['form'] = form return f(self, *args, **kwargs) return wrapper + +class _Mask: + EMAIL = r'(?<=[a-z]{2})\w+(?=@)' + EMAIL_ANONYMIZATION = r'^\w+(?=@)' + + def __call__(self, data, show_begin=0, show_end=0, /, *, + pattern=None, mask_char='*', n_mask_char=6): + if data is None: + return None + if pattern is None: + return data[:show_begin] + mask_char * n_mask_char \ + + data[len(data) - show_end:] + return re.sub(pattern, mask_char * n_mask_char, data) + +mask = _Mask() \ No newline at end of file From f358d9fb1fb02edd97cf81ffeed09c817bd58e2c Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Thu, 29 Oct 2020 19:42:00 -0300 Subject: [PATCH 02/12] Fix sql dump links and change language to Python3.8 --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b1a2b31..a4a1c51 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,14 @@ FROM postgres as dellstore -ADD https://git.io/JT1ee /docker-entrypoint-initdb.d/dellstore.sql +ADD https://git.io/JTdKr /docker-entrypoint-initdb.d/dellstore.sql RUN chmod 744 /docker-entrypoint-initdb.d/dellstore.sql FROM postgres as dellstore_isolated -# COPY sql/dellstore_isolated.sql /usr/src/dellstore_isolated.sql +ADD https://git.io/JTdKZ /docker-entrypoint-initdb.d/dellstore_isolated.sql +RUN chmod 744 /docker-entrypoint-initdb.d/dellstore_isolated.sql -FROM python:3.7 AS backend +FROM python:3.8 AS backend ARG PYTHON_DEBUG # Assign value 1 to variables in Debug Mode From 9d92347d07fb6286128cc8cd4a5f0ac61fd1bece Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Thu, 29 Oct 2020 20:13:00 -0300 Subject: [PATCH 03/12] Add libraries for encrypting the isolated database --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index b19824c..e6239dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ astroid==2.4.2 Babel==2.8.0 +cffi==1.14.3 click==7.1.2 colorama==0.4.3 +cryptography==3.2.1 debugpy==1.0.0 decorator==4.4.2 Flask==1.1.2 @@ -18,6 +20,7 @@ mccabe==0.6.1 passlib==1.7.4 phonenumbers==8.12.11 psycopg2==2.8.6 +pycparser==2.20 pylint==2.6.0 pytz==2020.1 six==1.15.0 From faa24111afacd7e5f4dc3fdece6d74a6410ae4ee Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Sat, 31 Oct 2020 10:13:00 -0300 Subject: [PATCH 04/12] Fix database URI & remove debuging port setting --- .env | 5 +---- settings.py | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.env b/.env index ee10125..8eb0547 100644 --- a/.env +++ b/.env @@ -13,9 +13,6 @@ ISOLATED_DB_USER=postgres # -- Used in by backend image on Dockerfile PYTHON_DEBUG=1 -# -- Used by the python debugger -PYTHON_DEBUG_PORT=10001 - # -- Flask Settings FLASK_APPSETTINGS=settings.Dev FLASK_ENV=development @@ -24,4 +21,4 @@ FLASK_PORT=80 # --- SQL Alchemy settings BASE_DB_URI=postgresql://postgres:{PASSWORD}@db:5432/store -BASE_ISOLATED_DB_URI=postgresql://postgres:{PASSWORD}@$db_isolated:5432/isolatedstore \ No newline at end of file +BASE_ISOLATED_DB_URI=postgresql://postgres:{PASSWORD}@db_isolated:5432/isolatedstore \ No newline at end of file diff --git a/settings.py b/settings.py index 53808db..24b3598 100644 --- a/settings.py +++ b/settings.py @@ -33,6 +33,4 @@ class Dev(Settings): DEBUG = True TESTING = True - SQLALCHEMY_ECHO = True - - PYTHON_DEBUG_PORT = os.environ['PYTHON_DEBUG_PORT'] \ No newline at end of file + SQLALCHEMY_ECHO = True \ No newline at end of file From 282a6c63d75c4dd731bd60a2163358b60973a71e Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Sat, 31 Oct 2020 12:35:00 -0300 Subject: [PATCH 05/12] Add mask anonymization to Customers model --- models/customers.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/models/customers.py b/models/customers.py index c0c742f..7f6942d 100644 --- a/models/customers.py +++ b/models/customers.py @@ -1,6 +1,8 @@ from datetime import datetime from flask.globals import request + +from helpers import mask from app import db import sqlalchemy_utils as su @@ -33,22 +35,18 @@ class Customers(db.Model): city = db.Column(db.String(50), info={'anonymize': True}) state = db.Column(db.String(50), info={'anonymize': True}) zip = db.Column(db.Integer, info={'anonymize': True}) - country = db.Column(su.CountryType, info={'anonymize': True}) + country = db.Column(su.CountryType) region = db.Column(db.Integer) email = db.Column(su.EmailType(50)) - phone = db.Column(su.PhoneNumberType(max_length=50), - info={'anonymize': True}) + phone = db.Column(su.PhoneNumberType(max_length=50)) creditcardtype = db.Column(db.Integer, info={'anonymize': True}) creditcard = db.Column(db.String(50), info={'anonymize': True}) - creditcardexpiration = db.Column(db.String(50), - info={'anonymize': True}) + creditcardexpiration = db.Column(db.String(50), info={'anonymize': True}) username = db.Column(db.String(50), info={'anonymize': True}) - password = db.Column(su.PasswordType(schemes=['pbkdf2_sha512']), - info={'anonymize': True}) + password = db.Column(su.PasswordType(schemes=['pbkdf2_sha512'])) age = db.Column(db.Integer, info={'anonymize': True}) income = db.Column(db.Integer, info={'anonymize': True}) - gender = db.Column(su.ChoiceType(GENDERS, - impl=db.String(1)), info={'anonymize': True}) + gender = db.Column(su.ChoiceType(GENDERS, impl=db.String(1))) _deleted_at = db.Column('deleted_at', db.DateTime) shopping_history = db.relationship('Orders', secondary='cust_hist') @@ -65,9 +63,12 @@ def from_form(cls, form): def anonymized(self): """Anonimiza informacoes que identificam uma pessoa""" - # TODO: Change anonymization process to - # TODO: Atualizar forma de aplicar data - self._deleted = datetime.now() + self._deleted_at = datetime.now() + self.phone = self.phone and mask(self.phone.e164[1:], 5, 0) + self.email = self.email and \ + mask(self.email, pattern=mask.EMAIL_ANONYMIZATION, n_mask_char=2) + self.password = None + for column in self.__table__.columns: if column.info.get('anonymize'): setattr(self, column.name, None) From cba211e1e7ab9ad065642fd01a3bd357fcf5cd65 Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Sat, 31 Oct 2020 14:56:00 -0300 Subject: [PATCH 06/12] Fix bug when cookie is set for a customer that is not active --- server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server.py b/server.py index c9d6c7c..37d1157 100644 --- a/server.py +++ b/server.py @@ -10,9 +10,10 @@ def assign_loggedin_customer(): customerid = session.get('customerid') if customerid is not None: customer = Customers.query.get(customerid) - if customer is not None: - g.user = customer if customer.is_active else None - + if customer is not None and customer.is_active: + g.user = customer + else: + session.clear() app.add_url_rule('/signin', view_func=Signin.as_view('signin')) app.add_url_rule('/signup', view_func=Signup.as_view('signup')) From 3d44be8ef29f4e5175dd5470faaa546fb278ac46 Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Sun, 1 Nov 2020 13:37:00 -0300 Subject: [PATCH 07/12] Add custom base Model for saving and commiting changes --- app.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index f6c6467..bc0de33 100644 --- a/app.py +++ b/app.py @@ -1,12 +1,22 @@ from flask import Flask from flask_sqlalchemy import SQLAlchemy +from flask_sqlalchemy.model import Model from werkzeug.utils import import_string import os +# Flask Settings + app = Flask(__name__) cfg = import_string(os.environ.get('FLASK_APPSETTINGS'))() app.config.from_object(cfg) -db = SQLAlchemy(app) \ No newline at end of file +# SQLAlchemy settings + +class BaseModel(Model): + def save(self): + db.session.add(self) + db.session.commit() + +db = SQLAlchemy(app, model_class=BaseModel) \ No newline at end of file From 930f88be84904abe4f3bb7482066d15f0b8d5c3a Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Sun, 1 Nov 2020 19:17:00 -0300 Subject: [PATCH 08/12] Add isolated database model with encrypted columns --- models/isolated/__init__.py | 2 ++ models/isolated/customer_personal_info.py | 44 +++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/models/isolated/__init__.py b/models/isolated/__init__.py index e69de29..506f462 100644 --- a/models/isolated/__init__.py +++ b/models/isolated/__init__.py @@ -0,0 +1,2 @@ +from .customer_personal_info import * + diff --git a/models/isolated/customer_personal_info.py b/models/isolated/customer_personal_info.py index d885e5f..ae5f970 100644 --- a/models/isolated/customer_personal_info.py +++ b/models/isolated/customer_personal_info.py @@ -1,5 +1,43 @@ +from flask.globals import session +from sqlalchemy_utils.types.encrypted.encrypted_type import AesEngine, FernetEngine +from sqlalchemy_utils import EncryptedType +from cryptography.fernet import Fernet + from app import db +from models.customers import Customers + + +def _get_customer_personal_info(model, ColumnType, engine, key): + + AnonymizedColumnsMixin = type('AnonymizedColumnsMixin', (), { + column.name: db.Column(ColumnType(column.type, key, engine)) + for column in model.__table__.columns + if column.info.get('anonymize') + }) + + class CustomerPersonalInfo(AnonymizedColumnsMixin, db.Model): + """ + Class writes deleted customer's personal info encrypted + on the isolated database + """ + __bind_key__ = 'db_isolated' + __tablename__ = 'customers_personal_info' + + customerid = db.Column('customerid', db.Integer, primary_key=True) + + @classmethod + def from_customer(cls, customer): + self = cls() + for column in self.__table__.columns: + setattr(self, column.name, getattr(customer, column.name)) + return self + + return CustomerPersonalInfo + + +def _get_key(): + session['cryptkey'] = session.get('cryptkey', Fernet.generate_key()) + return session['cryptkey'] -class CustomerPersonalInfo(db.Model): - __bind_key__ = 'db_isolated' - \ No newline at end of file +CustomerPersonalInfo = _get_customer_personal_info( + Customers, EncryptedType, FernetEngine, _get_key) \ No newline at end of file From fde0bc950e7491541ae872377f88e4bf0498d1d6 Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Mon, 2 Nov 2020 09:57:00 -0300 Subject: [PATCH 09/12] Change view to support the pseudonymization with isolated encrypted database --- views/account.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/views/account.py b/views/account.py index 08c0d79..0b99449 100644 --- a/views/account.py +++ b/views/account.py @@ -12,6 +12,7 @@ ) from forms.account import SignupForm, SigninForm, UpdateAccountForm from models.customers import Customers +from models.isolated import CustomerPersonalInfo from views.utils import (FormMethodView, MethodViewWrapper, RequiredLoggedoutViewMixin, RequiredLoginViewMixin, ) @@ -48,17 +49,26 @@ def get(self): class AccountDelete(MethodViewWrapper, RequiredLoginViewMixin): """Rota para anonimizar Customer""" - @commit_on_finish def get(self): if g.user is None: flash('Customer not found', category='error') abort(404) - - yield g.user.anonymized() + + user_personal_info = CustomerPersonalInfo.from_customer(g.user) + g.user.anonymized().save() + user_personal_info.save() + self.store_key_id(user_personal_info.customerid) flash("We're sorry to see you go :(") return redirect(url_for(Signout.ROUTE)) + def store_key_id(self, customerid): + """Store id;key on an isolated environment""" + # TODO: Change how to manage the keys + # something like KMS from Amazon + with open('isolated_db_keys.txt', 'a') as f: + f.write('%s;%s\n' % (customerid, session['cryptkey'].decode())) + class Signin(FormMethodView, RequiredLoggedoutViewMixin): """Rota de login""" From 301e593d8d0cd6462fd73046adf41de9e1c23029 Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Mon, 2 Nov 2020 12:57:00 -0300 Subject: [PATCH 10/12] Fix bug related to the orders of the decorators --- views/account.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/views/account.py b/views/account.py index 0b99449..4fe4caa 100644 --- a/views/account.py +++ b/views/account.py @@ -27,8 +27,8 @@ def get(self): # TODO: return super().get() - @commit_on_finish @form_validated_or_page_with_errors + @commit_on_finish def post(self, form=None): customer = Customers.from_form(form) yield customer # deletes customer @@ -96,8 +96,8 @@ class Signup(FormMethodView, RequiredLoggedoutViewMixin): FORM = SignupForm - @commit_on_finish @form_validated_or_page_with_errors + @commit_on_finish def post(self, form=None): # TODO: Verify if username doesn't exist already customer = Customers.from_form(form) From db9129726398a5ae0bdbd1839b7a335797798797 Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Tue, 3 Nov 2020 16:15:00 -0300 Subject: [PATCH 11/12] Phone and email masked on account view template for sprint-4 --- forms/account.py | 4 ++-- server.py | 9 +++++++++ templates/account.jinja | 11 ++++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/forms/account.py b/forms/account.py index c10aeb1..1689c57 100644 --- a/forms/account.py +++ b/forms/account.py @@ -15,12 +15,12 @@ class SignupForm(ModelForm): class Meta: model = Customers only = [ - 'firstname', 'lastname', + 'email', 'phone', 'firstname', 'lastname', 'gender', 'age', 'income', 'country', 'zip', 'city', 'state', 'address1', 'address2', 'creditcard', 'creditcardexpiration', - 'phone', 'username', 'email', 'password', + 'username', 'password', ] class UpdateAccountForm(SignupForm): diff --git a/server.py b/server.py index 37d1157..746584b 100644 --- a/server.py +++ b/server.py @@ -3,6 +3,8 @@ from views import * +from helpers import mask + @app.before_request def assign_loggedin_customer(): @@ -14,6 +16,13 @@ def assign_loggedin_customer(): g.user = customer else: session.clear() + +# TODO: for sprint-4. To be refactored +@app.context_processor +def utility_processor(): + """Pass mask function to jinja""" + return dict(mask=mask) + app.add_url_rule('/signin', view_func=Signin.as_view('signin')) app.add_url_rule('/signup', view_func=Signup.as_view('signup')) diff --git a/templates/account.jinja b/templates/account.jinja index 80ee742..e73410a 100644 --- a/templates/account.jinja +++ b/templates/account.jinja @@ -12,8 +12,17 @@ + {# TODO: Refactor this uglyness. It works for now :) #} {% for field in form %} - + {% if field.label.text == 'gender' %} + + {% elif field.label.text == 'email' %} + + {% elif field.label.text == 'phone' %} + + {% else %} + + {% endif %} {% endfor %}
{{ field.label.text|upper }}{{ field.data }}
{{ field.label.text|upper }}{{ field.object_data.value or "" }}
{{ field.label.text|upper }}{{ mask(field.data, pattern=mask.EMAIL) }}
{{ field.label.text|upper }}{{ mask(field.data.international, 4, 2) }}
{{ field.label.text|upper }}{{ field.data or "" }}
From 824af91e7d1d088660f27abeee329db5c0ca9626 Mon Sep 17 00:00:00 2001 From: Maicon Mauricio <64911189+maiconandsilva@users.noreply.github.com> Date: Wed, 4 Nov 2020 01:15:00 -0300 Subject: [PATCH 12/12] Update README.md for sprint-4 --- README.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 16ef3a4..2f24861 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ docker-compose up --build -d docker-compose up -d # URL de acesso -url: http://127.0.0.1:5000/tests +url: http://127.0.0.1/signup ``` ## :beers: Contribuições @@ -67,17 +67,23 @@ divididas em sprints, e para controle dessas, foram utilizados ferramentas como - [x] Criptografia da senha do usuário - [x] Rota de criar usuário - [x] Botão para ativar rota de anonimização +- [x] Autenticação +- [x] Telas de interação do usuário - [x] BurnDown / Velocity Chart ### Sprint 4 -- [ ] Máscara para o e-mail -- [ ] Máscara para o Cartão de crédito -- [ ] Máscara para dados como o telefone -- [ ] Validação dos campos do usuário +- [x] Aplicar máscaras ao cliente deletado no banco para campos o e-mail, cartão de crédito e telefone +- [x] Aplicar máscaras para visualização de informações pessoais do cliente em sua conta +- [x] Pseudonimização (transferir dados do cliente para outra tabela) +- [x] Criptografar dados pessoais do cliente "pseudonimizado" ### Sprint 5 -- [ ] Autenticação -- [ ] Telas de interação do usuário +- [ ] Implementar estrutura para guardar chaves advindas da Pseudonimização do cliente +- [ ] Implementar autorização para clientes com senhas antes não criptografadas +- [ ] Validação dos campos do usuário + +### Sprint 6 +- [ ] Implementar tela de visualização de produtos ## :anchor: Fontes - Exemplo da Empresa Dell Store de [Banco de dados ](https://linux.dell.com/dvdstore/) utilizado para anonimização.