From 078acecda2c05af2a8bd9227978b07071f6b6324 Mon Sep 17 00:00:00 2001 From: Viggo de Vries Date: Tue, 12 Dec 2023 14:15:20 +0100 Subject: [PATCH 1/6] klunk --- Makefile | 1 + .../management/commands/test_illshit.py | 12 ++-- oscar_odin/mappings/context.py | 6 +- oscar_odin/resources/catalogue.py | 2 + tests/test_settings.py | 71 +++++++++++++++++++ 5 files changed, 84 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index b6dcc63..60dcedf 100644 --- a/Makefile +++ b/Makefile @@ -33,4 +33,5 @@ black: ill: rm db.sqlite3 cp klaas.sqlite3 db.sqlite3 + python3 runtests.py migrate python3 runtests.py test_illshit diff --git a/oscar_odin/management/commands/test_illshit.py b/oscar_odin/management/commands/test_illshit.py index bcdb6dc..cc40162 100644 --- a/oscar_odin/management/commands/test_illshit.py +++ b/oscar_odin/management/commands/test_illshit.py @@ -67,9 +67,9 @@ def handle(self, *args, **options): partner, _ = Partner.objects.get_or_create(name="klaas") - Category.add_root(name="Hatsie", slug="batsie", is_public=True) - Category.add_root(name="henk", slug="klaas", is_public=True) - Category.add_root(name="Knaken", slug="knaken", is_public=True) + batsie = Category.add_root(name="Hatsie", slug="batsie", is_public=True, identifier="batsie") + henk = batsie.add_child(name="henk", slug="klaas", is_public=True, identifier="henk") + henk.add_child(name="Knaken", slug="knaken", is_public=True, identifier="knaken") products = [] @@ -95,9 +95,9 @@ def handle(self, *args, **options): ImageResource(caption="gekke caption", display_order=0, original=File(output, name="image%s.jpg")), ], categories=[ - CategoryResource(name="henk", slug="klaas"), - CategoryResource(name="Hatsie datsie", slug="batsie"), - CategoryResource(name="Knaken", slug="knaken") + CategoryResource(identifier="batsie"), + CategoryResource(identifier="henk"), + CategoryResource(identifier="knaken") ], attributes=attributes ) diff --git a/oscar_odin/mappings/context.py b/oscar_odin/mappings/context.py index 355f637..5917f23 100644 --- a/oscar_odin/mappings/context.py +++ b/oscar_odin/mappings/context.py @@ -11,13 +11,15 @@ Category = get_model("catalogue", "Category") StockRecord = get_model("partner", "StockRecord") ProductClass = get_model("catalogue", "ProductClass") +ProductImage = get_model("catalogue", "ProductImage") MODEL_IDENTIFIERS_MAPPING = { - Category: ("slug",), + Category: ("identifier",), Product: ("upc",), - StockRecord: ("product_id", "partner_id", "partner_sku"), + StockRecord: ("product_id",), ProductClass: ("slug",), + ProductImage: ("identifier",), } diff --git a/oscar_odin/resources/catalogue.py b/oscar_odin/resources/catalogue.py index 2a127f1..614c96c 100644 --- a/oscar_odin/resources/catalogue.py +++ b/oscar_odin/resources/catalogue.py @@ -31,6 +31,7 @@ class Meta: verbose_name_plural = "Product images" id: int + identifier: str original: Any caption: str = odin.Options(empty=True) display_order: int = odin.Options( @@ -47,6 +48,7 @@ class Category(OscarCatalogue): """A category within Django Oscar.""" id: int + identifier: str name: str slug: str description: str diff --git a/tests/test_settings.py b/tests/test_settings.py index 1f4e7d4..07ebcba 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,3 +1,11 @@ +import os +import oscar + +# Path helper +location = lambda x: os.path.join( + os.path.dirname(os.path.realpath(__file__)), x) + + DEBUG = True USE_TZ = True SITE_ID = 1 @@ -11,6 +19,37 @@ SECRET_KEY = "123" +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + location('templates'), + ], + 'OPTIONS': { + 'loaders': [ + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', + ], + 'context_processors': [ + 'django.contrib.auth.context_processors.auth', + 'django.template.context_processors.request', + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.contrib.messages.context_processors.messages', + + # Oscar specific + 'oscar.apps.search.context_processors.search_form', + 'oscar.apps.communication.notifications.context_processors.notifications', + 'oscar.apps.checkout.context_processors.checkout', + 'oscar.core.context_processors.metadata', + ], + 'debug': DEBUG, + } + } +] + INSTALLED_APPS = [ "django.contrib.auth", "django.contrib.contenttypes", @@ -18,6 +57,15 @@ "django.contrib.staticfiles", "django.contrib.sites", "django.contrib.flatpages", + "django.contrib.admin", + "django.contrib.messages", + 'widget_tweaks', + 'haystack', + 'treebeard', + 'sorl.thumbnail', + 'django_tables2', + + # Oscar apps "oscar.config.Shop", "oscar.apps.analytics.apps.AnalyticsConfig", @@ -53,6 +101,25 @@ "oscar_odin.apps.OscarOdinAppConfig", ] +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware', + + # Allow languages to be selected + 'django.middleware.locale.LocaleMiddleware', + 'django.middleware.http.ConditionalGetMiddleware', + 'django.middleware.common.CommonMiddleware', + + # Ensure a valid basket is added to the request instance for every request + 'oscar.apps.basket.middleware.BasketMiddleware', +] + from oscar.defaults import * HAYSTACK_CONNECTIONS = { @@ -62,3 +129,7 @@ } SILENCED_SYSTEM_CHECKS = ["models.W042"] + +STATIC_URL = "/static/" + +ROOT_URLCONF = "urls" \ No newline at end of file From fa594019b21d836f68e93d7b72fb50c7df06fd8e Mon Sep 17 00:00:00 2001 From: Viggo de Vries Date: Thu, 14 Dec 2023 12:03:36 +0100 Subject: [PATCH 2/6] Work with new constants --- Makefile | 3 +- .../management/commands/test_illshit.py | 29 +++---- oscar_odin/mappings/catalogue.py | 4 +- oscar_odin/mappings/constants.py | 78 +++++++++++++++++++ oscar_odin/mappings/context.py | 16 ++-- oscar_odin/mappings/defaults.py | 12 +-- oscar_odin/resources/catalogue.py | 4 +- oscar_odin/utils.py | 4 +- tests/reverse/test_catalogue.py | 15 ++-- 9 files changed, 124 insertions(+), 41 deletions(-) create mode 100644 oscar_odin/mappings/constants.py diff --git a/Makefile b/Makefile index 60dcedf..dd1d084 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,7 @@ lint: fail-if-no-virtualenv pylint oscar_odin/ test: fail-if-no-virtualenv - python3 runtests.py makemigrations --check --dry-run - @python3 runtests.py test tests/ + python3 runtests.py test tests/ black: @black oscar_odin/**/*.py diff --git a/oscar_odin/management/commands/test_illshit.py b/oscar_odin/management/commands/test_illshit.py index cc40162..47836dc 100644 --- a/oscar_odin/management/commands/test_illshit.py +++ b/oscar_odin/management/commands/test_illshit.py @@ -18,6 +18,8 @@ ) from oscar_odin.mappings.defaults import DEFAULT_UPDATE_FIELDS +from oscar_odin.mappings.constants import * + from oscar_odin.utils import querycounter Product = get_model("catalogue", "Product") @@ -64,22 +66,22 @@ def handle(self, *args, **options): product_class = ProductClassResource( slug="klaas", name="Klaas" ) - + partner, _ = Partner.objects.get_or_create(name="klaas") - batsie = Category.add_root(name="Hatsie", slug="batsie", is_public=True, identifier="batsie") - henk = batsie.add_child(name="henk", slug="klaas", is_public=True, identifier="henk") - henk.add_child(name="Knaken", slug="knaken", is_public=True, identifier="knaken") + batsie = Category.add_root(name="Hatsie", slug="batsie", is_public=True, code="batsie") + henk = batsie.add_child(name="henk", slug="klaas", is_public=True, code="henk") + henk.add_child(name="Knaken", slug="knaken", is_public=True, code="knaken") products = [] - - for i in range(0, 5000): + + def create_product(i): attributes = dict() attributes.update({code: "%s-%s" % (code, i) for code in text_codes}) attributes.update({code: i for code in int_codes}) attributes.update({code: option for code in option_codes}) - products.append(ProductResource( + return ProductResource( upc="1234323-%s" % i, title="asdf2 %s" % i, slug="asdf-asdfasdf-%s" % i, @@ -95,15 +97,16 @@ def handle(self, *args, **options): ImageResource(caption="gekke caption", display_order=0, original=File(output, name="image%s.jpg")), ], categories=[ - CategoryResource(identifier="batsie"), - CategoryResource(identifier="henk"), - CategoryResource(identifier="knaken") + CategoryResource(code="batsie"), + CategoryResource(code="henk"), + CategoryResource(code="knaken") ], attributes=attributes ) - ) - + + products = list(map(create_product, range(0, 5000))) + with querycounter("COMMANDO"): - products_to_db(products) + products_to_db(products, fields_to_update=ALL_PRODUCT_FIELDS + ALL_STOCKRECORD_FIELDS+ ALL_PRODUCTIMAGE_FIELDS) print("AANTAL PRODUCTEN AANGEMAAKT:", Product.objects.count()) diff --git a/oscar_odin/mappings/catalogue.py b/oscar_odin/mappings/catalogue.py index 1ec059b..6ffa58b 100644 --- a/oscar_odin/mappings/catalogue.py +++ b/oscar_odin/mappings/catalogue.py @@ -26,7 +26,7 @@ save_foreign_keys, ) from .context import ModelMapperContext -from .defaults import DEFAULT_UPDATE_FIELDS +from .constants import ALL_CATALOGUE_FIELDS __all__ = ( "ProductImageToResource", @@ -392,7 +392,7 @@ def products_to_model( def products_to_db( products: List[resources.catalogue.Product], rollback=True, - fields_to_update=DEFAULT_UPDATE_FIELDS, + fields_to_update=ALL_CATALOGUE_FIELDS, ) -> Tuple[List[ProductModel], Dict]: """Map mulitple products to a model and store them in the database. diff --git a/oscar_odin/mappings/constants.py b/oscar_odin/mappings/constants.py new file mode 100644 index 0000000..f207bf9 --- /dev/null +++ b/oscar_odin/mappings/constants.py @@ -0,0 +1,78 @@ +PRODUCT_STRUCTURE = "Product.structure" +PRODUCT_IS_PUBLIC = "Product.is_public" +PRODUCT_UPC = "Product.upc" +PRODUCT_PARENT = "Product.parent" +PRODUCT_TITLE = "Product.title" +PRODUCT_SLUG = "Product.slug" +PRODUCT_DESCRIPTION = "Product.description" +PRODUCT_META_TITLE = "Product.meta_title" +PRODUCT_META_DESCRIPTION = "Product.meta_description" +PRODUCT_PRODUCT_CLASS = "Product.product_class" +PRODUCT_IS_DISCOUNTABLE = "Product.is_discountable" + +CATEGORY_NAME = "Category.name" +CATEGORY_CODE = "Category.code" +CATEGORY_DESCRIPTION = "Category.description" +CATEGORY_META_TITLE = "Category.meta_title" +CATEGORY_META_DESCRIPTION = "Category.meta_description" +CATEGORY_IMAGE = "Category.image" +CATEGORY_SLUG = "Category.slug" +CATEGORY_IS_PUBLIC = "Category.is_public" + +PRODUCTIMAGE_CODE = "ProductImage.code" +PRODUCTIMAGE_ORIGINAL = "ProductImage.original" +PRODUCTIMAGE_CAPTION = "ProductImage.caption" +PRODUCTIMAGE_DISPLAY_ORDER = "ProductImage.display_order" + +STOCKRECORD_PARTNER = "StockRecord.partner" +STOCKRECORD_PARTNER_SKU = "StockRecord.partner_sku" +STOCKRECORD_PRICE_CURRENCY = "StockRecord.price_currency" +STOCKRECORD_PRICE = "StockRecord.price" +STOCKRECORD_NUM_IN_STOCK = "StockRecord.num_in_stock" +STOCKRECORD_NUM_ALLOCATED = "StockRecord.num_allocated" + +ALL_PRODUCT_FIELDS = [ + PRODUCT_STRUCTURE, + PRODUCT_IS_PUBLIC, + PRODUCT_UPC, + PRODUCT_PARENT, + PRODUCT_TITLE, + PRODUCT_SLUG, + PRODUCT_DESCRIPTION, + PRODUCT_META_TITLE, + PRODUCT_META_DESCRIPTION, + PRODUCT_PRODUCT_CLASS, + PRODUCT_IS_DISCOUNTABLE, +] + +ALL_CATEGORY_FIELDS = [ + CATEGORY_NAME, + CATEGORY_CODE, + CATEGORY_DESCRIPTION, + CATEGORY_META_TITLE, + CATEGORY_META_DESCRIPTION, + CATEGORY_IMAGE, + CATEGORY_IS_PUBLIC, + CATEGORY_SLUG, +] + +ALL_PRODUCTIMAGE_FIELDS = [ + PRODUCTIMAGE_CODE, + PRODUCTIMAGE_ORIGINAL, + PRODUCTIMAGE_CAPTION, + PRODUCTIMAGE_DISPLAY_ORDER, +] + +ALL_STOCKRECORD_FIELDS = [ + STOCKRECORD_PARTNER, + STOCKRECORD_PARTNER_SKU, + STOCKRECORD_PRICE_CURRENCY, + STOCKRECORD_PRICE, + STOCKRECORD_NUM_IN_STOCK, + STOCKRECORD_NUM_ALLOCATED, +] + + +ALL_CATALOGUE_FIELDS = ( + ALL_PRODUCT_FIELDS + ALL_PRODUCTIMAGE_FIELDS + ALL_STOCKRECORD_FIELDS +) diff --git a/oscar_odin/mappings/context.py b/oscar_odin/mappings/context.py index 5917f23..ae564aa 100644 --- a/oscar_odin/mappings/context.py +++ b/oscar_odin/mappings/context.py @@ -15,11 +15,11 @@ MODEL_IDENTIFIERS_MAPPING = { - Category: ("identifier",), + Category: ("code",), Product: ("upc",), StockRecord: ("product_id",), ProductClass: ("slug",), - ProductImage: ("identifier",), + ProductImage: ("code",), } @@ -51,7 +51,7 @@ def get_instances_to_create_or_update(Model, instances): for instance in instances: key = get_key_values(instance) - if isinstance(key, str): + if not isinstance(key, tuple): key = (key,) if key in id_mapping: @@ -64,7 +64,7 @@ def get_instances_to_create_or_update(Model, instances): return instances_to_create, instances_to_update else: - return instances, instances_to_update + return instances, [] class ModelMapperContext(dict): @@ -107,8 +107,12 @@ def add_fields_to_update(self, fields_to_update): self.fields_to_update = fields_to_update def get_fields_to_update(self, Model): - model_field_names = [field.name for field in Model._meta.get_fields()] - return [f for f in self.fields_to_update if f in model_field_names] or None + modelname = "%s." % Model.__name__ + return [ + f.replace(modelname, "") + for f in self.fields_to_update + if f.startswith(modelname) + ] or None def get_create_and_update_relations(self, related_instance_items): to_create = defaultdict(list) diff --git a/oscar_odin/mappings/defaults.py b/oscar_odin/mappings/defaults.py index 547d22b..7fe95c8 100644 --- a/oscar_odin/mappings/defaults.py +++ b/oscar_odin/mappings/defaults.py @@ -1,9 +1,9 @@ DEFAULT_UPDATE_FIELDS = [ - "title", - "slug", - "price", - "num_in_stock", - "num_allocated", - "name", + "product.title", + "product.slug", + "product.price", + "stockrecord.num_in_stock", + "stockrecord.num_allocated", + "category.name", "display_order", ] diff --git a/oscar_odin/resources/catalogue.py b/oscar_odin/resources/catalogue.py index 614c96c..84c784f 100644 --- a/oscar_odin/resources/catalogue.py +++ b/oscar_odin/resources/catalogue.py @@ -31,7 +31,7 @@ class Meta: verbose_name_plural = "Product images" id: int - identifier: str + code: str original: Any caption: str = odin.Options(empty=True) display_order: int = odin.Options( @@ -48,7 +48,7 @@ class Category(OscarCatalogue): """A category within Django Oscar.""" id: int - identifier: str + code: str name: str slug: str description: str diff --git a/oscar_odin/utils.py b/oscar_odin/utils.py index 729e936..d0d91fd 100644 --- a/oscar_odin/utils.py +++ b/oscar_odin/utils.py @@ -50,10 +50,10 @@ def in_bulk(self, instances=None, field_names=('pk',)): for offset in range(0, len(instances), batch_size): batch = instances[offset:offset + batch_size] query = get_query(batch, field_names) - qs += tuple(self.filter(query).order_by().values(*("pk",) + field_names)) + qs += tuple(self.filter(query).order_by().values("pk", *field_names)) else: query = get_query(instances, field_names) - qs = self.filter(query).order_by().values(*("pk",) + field_names) + qs = self.filter(query).order_by().values("pk", *field_names) else: return {} diff --git a/tests/reverse/test_catalogue.py b/tests/reverse/test_catalogue.py index ed36d59..b884c0d 100644 --- a/tests/reverse/test_catalogue.py +++ b/tests/reverse/test_catalogue.py @@ -17,7 +17,7 @@ ProductAttributeValue as ProductAttributeValueResource ) -from oscar_odin.mappings.defaults import DEFAULT_UPDATE_FIELDS +from oscar_odin.mappings.constants import STOCKRECORD_PRICE, STOCKRECORD_NUM_IN_STOCK, STOCKRECORD_NUM_ALLOCATED, PRODUCTIMAGE_ORIGINAL Product = get_model("catalogue", "Product") ProductClass = get_model("catalogue", "ProductClass") @@ -54,8 +54,8 @@ def test_create_product_with_related_fields(self): partner = Partner.objects.create(name="klaas") - Category.add_root(name="Hatsie", slug="batsie", is_public=True) - Category.add_root(name="henk", slug="klaas", is_public=True) + Category.add_root(name="Hatsie", slug="batsie", is_public=True, code="1") + Category.add_root(name="henk", slug="klaas", is_public=True, code="2") product_resource = ProductResource( upc="1234323-2", @@ -73,7 +73,7 @@ def test_create_product_with_related_fields(self): ImageResource(caption="gekke caption", display_order=0, original=File(self.image, name="harrie.jpg")), ImageResource(caption="gekke caption 2", display_order=1, original=File(self.image, name="vats.jpg")), ], - categories=[CategoryResource(name="henk", slug="klaas"), CategoryResource(name="Hatsie datsie", slug="batsie")], + categories=[CategoryResource(code="2")], attributes={"henk": "Klaas", "harrie": 1} ) @@ -82,10 +82,9 @@ def test_create_product_with_related_fields(self): prd = Product.objects.get(upc="1234323-2") self.assertEquals(prd.title, "asdf2") - self.assertEquals(Category.objects.get(slug="batsie").name, "Hatsie datsie") self.assertEquals(prd.images.count(), 2) self.assertEquals(Category.objects.count(), 2) - self.assertEquals(prd.categories.count(), 2) + self.assertEquals(prd.categories.count(), 1) self.assertEquals(prd.stockrecords.count(), 1) stockrecord = prd.stockrecords.first() @@ -108,10 +107,10 @@ def test_create_product_with_related_fields(self): ImageResource(caption="gekke caption", display_order=0, original=File(self.image, name="harriebatsie.jpg")), ImageResource(caption="gekke caption 2", display_order=1, original=File(self.image, name="vatsie.jpg")), ], - categories=[CategoryResource(name="henk", slug="klaas"), CategoryResource(name="Hatsie datsie", slug="batsie")], + categories=[CategoryResource(code="1")], ) - fields_to_update = ["upc", "price", "num_in_stock", "num_allocated", "original"] + fields_to_update = [STOCKRECORD_PRICE, STOCKRECORD_NUM_IN_STOCK, STOCKRECORD_NUM_ALLOCATED, PRODUCTIMAGE_ORIGINAL] products_to_db(product_resource, fields_to_update=fields_to_update) prd = Product.objects.get(upc="1234323-2") From 7696489b682c38daea779fd28643fb911b3238c9 Mon Sep 17 00:00:00 2001 From: Viggo de Vries Date: Fri, 15 Dec 2023 11:12:32 +0100 Subject: [PATCH 3/6] Make attribute updates work --- Makefile | 4 +- .../management/commands/test_illshit.py | 53 ++-- oscar_odin/mappings/utils.py | 9 +- oscar_odin/utils.py | 13 +- tests/resources/test_catalogue.py | 2 +- tests/reverse/test_catalogue.py | 248 +++++++++++++++--- tests/test_settings.py | 85 +++--- 7 files changed, 300 insertions(+), 114 deletions(-) diff --git a/Makefile b/Makefile index dd1d084..a67d517 100644 --- a/Makefile +++ b/Makefile @@ -26,8 +26,8 @@ test: fail-if-no-virtualenv python3 runtests.py test tests/ black: - @black oscar_odin/**/*.py - + @black oscar_odin/ + @black tests/ ill: rm db.sqlite3 diff --git a/oscar_odin/management/commands/test_illshit.py b/oscar_odin/management/commands/test_illshit.py index 47836dc..74e5d69 100644 --- a/oscar_odin/management/commands/test_illshit.py +++ b/oscar_odin/management/commands/test_illshit.py @@ -14,7 +14,7 @@ Image as ImageResource, ProductClass as ProductClassResource, Category as CategoryResource, - ProductAttributeValue as ProductAttributeValueResource + ProductAttributeValue as ProductAttributeValueResource, ) from oscar_odin.mappings.defaults import DEFAULT_UPDATE_FIELDS @@ -34,47 +34,57 @@ class Command(BaseCommand): - def handle(self, *args, **options): img = PIL.Image.new(mode="RGB", size=(200, 200)) output = io.BytesIO() img.save(output, "jpeg") product_class, _ = ProductClass.objects.get_or_create( - slug="klaas", defaults={"requires_shipping": True, "track_stock": True, "name": "Klaas"} + slug="klaas", + defaults={"requires_shipping": True, "track_stock": True, "name": "Klaas"}, ) text_codes = ["code%s" % i for i in range(0, 10)] int_codes = ["code%s" % i for i in range(11, 20)] option_codes = ["code%s" % i for i in range(21, 30)] - + group, _ = AttributeOptionGroup.objects.get_or_create(name="gekke options") option, _ = AttributeOption.objects.get_or_create(group=group, option="klaas") - + for code in text_codes: ProductAttribute.objects.get_or_create( - name=code, code=code, type=ProductAttribute.TEXT, product_class=product_class + name=code, + code=code, + type=ProductAttribute.TEXT, + product_class=product_class, ) for code in int_codes: ProductAttribute.objects.get_or_create( - name=code, code=code, type=ProductAttribute.INTEGER, product_class=product_class + name=code, + code=code, + type=ProductAttribute.INTEGER, + product_class=product_class, ) for code in option_codes: ProductAttribute.objects.get_or_create( - name=code, code=code, type=ProductAttribute.OPTION, product_class=product_class, option_group=group + name=code, + code=code, + type=ProductAttribute.OPTION, + product_class=product_class, + option_group=group, ) - product_class = ProductClassResource( - slug="klaas", name="Klaas" - ) + product_class = ProductClassResource(slug="klaas", name="Klaas") partner, _ = Partner.objects.get_or_create(name="klaas") - batsie = Category.add_root(name="Hatsie", slug="batsie", is_public=True, code="batsie") + batsie = Category.add_root( + name="Hatsie", slug="batsie", is_public=True, code="batsie" + ) henk = batsie.add_child(name="henk", slug="klaas", is_public=True, code="henk") henk.add_child(name="Knaken", slug="knaken", is_public=True, code="knaken") products = [] - + def create_product(i): attributes = dict() attributes.update({code: "%s-%s" % (code, i) for code in text_codes}) @@ -94,19 +104,28 @@ def create_product(i): partner=partner, product_class=product_class, images=[ - ImageResource(caption="gekke caption", display_order=0, original=File(output, name="image%s.jpg")), + ImageResource( + caption="gekke caption", + display_order=0, + original=File(output, name="image%s.jpg"), + ), ], categories=[ CategoryResource(code="batsie"), CategoryResource(code="henk"), - CategoryResource(code="knaken") + CategoryResource(code="knaken"), ], - attributes=attributes + attributes=attributes, ) products = list(map(create_product, range(0, 5000))) with querycounter("COMMANDO"): - products_to_db(products, fields_to_update=ALL_PRODUCT_FIELDS + ALL_STOCKRECORD_FIELDS+ ALL_PRODUCTIMAGE_FIELDS) + products_to_db( + products, + fields_to_update=ALL_PRODUCT_FIELDS + + ALL_STOCKRECORD_FIELDS + + ALL_PRODUCTIMAGE_FIELDS, + ) print("AANTAL PRODUCTEN AANGEMAAKT:", Product.objects.count()) diff --git a/oscar_odin/mappings/utils.py b/oscar_odin/mappings/utils.py index c059857..786f220 100644 --- a/oscar_odin/mappings/utils.py +++ b/oscar_odin/mappings/utils.py @@ -117,6 +117,7 @@ def save_attributes(context, errors): fields_to_be_updated = set() for product, attr in context.attribute_data: + product.attr.invalidate() ( to_be_deleted, update_fields, @@ -128,7 +129,7 @@ def save_attributes(context, errors): attributes_to_delete.extend(to_be_deleted) if update_fields: - fields_to_be_updated.add(update_fields) + fields_to_be_updated.update(update_fields) if to_be_updated: attributes_to_update.extend(to_be_updated) @@ -137,13 +138,13 @@ def save_attributes(context, errors): attributes_to_create.extend(to_be_created) # now save all the attributes in bulk - if to_be_deleted: + if attributes_to_delete: ProductAttributeValue.objects.filter(pk__in=attributes_to_delete).delete() - if to_be_updated: + if attributes_to_update: ProductAttributeValue.objects.bulk_update( attributes_to_update, fields_to_be_updated, batch_size=500 ) - if to_be_created: + if attributes_to_create: ProductAttributeValue.objects.bulk_create( attributes_to_create, batch_size=500, ignore_conflicts=False ) diff --git a/oscar_odin/utils.py b/oscar_odin/utils.py index d0d91fd..e1b8987 100644 --- a/oscar_odin/utils.py +++ b/oscar_odin/utils.py @@ -27,11 +27,11 @@ def get_query(instances, field_names): query = filters.pop() for query_filter in filters: query = query | query_filter - + return query -def in_bulk(self, instances=None, field_names=('pk',)): +def in_bulk(self, instances=None, field_names=("pk",)): """ Return a dictionary mapping each of the given IDs to the object with that ID. If `id_list` isn't provided, evaluate the entire QuerySet. @@ -43,12 +43,17 @@ def in_bulk(self, instances=None, field_names=('pk',)): if not instances: return {} - batch_size = math.floor(connections[self.db].features.max_query_params / len(field_names)) - 1 + batch_size = ( + math.floor( + connections[self.db].features.max_query_params / len(field_names) + ) + - 1 + ) if batch_size and batch_size < len(instances): qs = () for offset in range(0, len(instances), batch_size): - batch = instances[offset:offset + batch_size] + batch = instances[offset : offset + batch_size] query = get_query(batch, field_names) qs += tuple(self.filter(query).order_by().values("pk", *field_names)) else: diff --git a/tests/resources/test_catalogue.py b/tests/resources/test_catalogue.py index 73950ab..82db561 100644 --- a/tests/resources/test_catalogue.py +++ b/tests/resources/test_catalogue.py @@ -13,7 +13,7 @@ Image as ImageResource, ProductClass as ProductClassResource, Category as CategoryResource, - ProductAttributeValue as ProductAttributeValueResource + ProductAttributeValue as ProductAttributeValueResource, ) from oscar_odin.mappings.defaults import DEFAULT_UPDATE_FIELDS diff --git a/tests/reverse/test_catalogue.py b/tests/reverse/test_catalogue.py index b884c0d..261175e 100644 --- a/tests/reverse/test_catalogue.py +++ b/tests/reverse/test_catalogue.py @@ -14,10 +14,15 @@ Image as ImageResource, ProductClass as ProductClassResource, Category as CategoryResource, - ProductAttributeValue as ProductAttributeValueResource + ProductAttributeValue as ProductAttributeValueResource, ) -from oscar_odin.mappings.constants import STOCKRECORD_PRICE, STOCKRECORD_NUM_IN_STOCK, STOCKRECORD_NUM_ALLOCATED, PRODUCTIMAGE_ORIGINAL +from oscar_odin.mappings.constants import ( + STOCKRECORD_PRICE, + STOCKRECORD_NUM_IN_STOCK, + STOCKRECORD_NUM_ALLOCATED, + PRODUCTIMAGE_ORIGINAL, +) Product = get_model("catalogue", "Product") ProductClass = get_model("catalogue", "ProductClass") @@ -29,7 +34,6 @@ class SingleProductReverseTest(TestCase): - @property def image(self): img = PIL.Image.new(mode="RGB", size=(200, 200)) @@ -42,16 +46,20 @@ def test_create_product_with_related_fields(self): name="Klaas", slug="klaas", requires_shipping=True, track_stock=True ) ProductAttribute.objects.create( - name="Henk", code="henk", type=ProductAttribute.TEXT, product_class=product_class + name="Henk", + code="henk", + type=ProductAttribute.TEXT, + product_class=product_class, ) ProductAttribute.objects.create( - name="Harrie", code="harrie", type=ProductAttribute.INTEGER, product_class=product_class - ) - - product_class = ProductClassResource( - slug="klaas", name="Klaas" + name="Harrie", + code="harrie", + type=ProductAttribute.INTEGER, + product_class=product_class, ) - + + product_class = ProductClassResource(slug="klaas", name="Klaas") + partner = Partner.objects.create(name="klaas") Category.add_root(name="Hatsie", slug="batsie", is_public=True, code="1") @@ -70,11 +78,19 @@ def test_create_product_with_related_fields(self): partner=partner, product_class=product_class, images=[ - ImageResource(caption="gekke caption", display_order=0, original=File(self.image, name="harrie.jpg")), - ImageResource(caption="gekke caption 2", display_order=1, original=File(self.image, name="vats.jpg")), + ImageResource( + caption="gekke caption", + display_order=0, + original=File(self.image, name="harrie.jpg"), + ), + ImageResource( + caption="gekke caption 2", + display_order=1, + original=File(self.image, name="vats.jpg"), + ), ], categories=[CategoryResource(code="2")], - attributes={"henk": "Klaas", "harrie": 1} + attributes={"henk": "Klaas", "harrie": 1}, ) prd = products_to_db(product_resource) @@ -85,7 +101,7 @@ def test_create_product_with_related_fields(self): self.assertEquals(prd.images.count(), 2) self.assertEquals(Category.objects.count(), 2) self.assertEquals(prd.categories.count(), 1) - + self.assertEquals(prd.stockrecords.count(), 1) stockrecord = prd.stockrecords.first() self.assertEquals(stockrecord.price, D("20")) @@ -93,7 +109,7 @@ def test_create_product_with_related_fields(self): self.assertEquals(prd.attr.henk, "Klaas") self.assertEquals(prd.attr.harrie, 1) - + self.assertEquals(prd.images.count(), 2) product_resource = ProductResource( @@ -104,13 +120,26 @@ def test_create_product_with_related_fields(self): partner=partner, product_class=product_class, images=[ - ImageResource(caption="gekke caption", display_order=0, original=File(self.image, name="harriebatsie.jpg")), - ImageResource(caption="gekke caption 2", display_order=1, original=File(self.image, name="vatsie.jpg")), + ImageResource( + caption="gekke caption", + display_order=0, + original=File(self.image, name="harriebatsie.jpg"), + ), + ImageResource( + caption="gekke caption 2", + display_order=1, + original=File(self.image, name="vatsie.jpg"), + ), ], categories=[CategoryResource(code="1")], ) - - fields_to_update = [STOCKRECORD_PRICE, STOCKRECORD_NUM_IN_STOCK, STOCKRECORD_NUM_ALLOCATED, PRODUCTIMAGE_ORIGINAL] + + fields_to_update = [ + STOCKRECORD_PRICE, + STOCKRECORD_NUM_IN_STOCK, + STOCKRECORD_NUM_ALLOCATED, + PRODUCTIMAGE_ORIGINAL, + ] products_to_db(product_resource, fields_to_update=fields_to_update) prd = Product.objects.get(upc="1234323-2") @@ -119,9 +148,131 @@ def test_create_product_with_related_fields(self): self.assertEquals(stockrecord.price, D("21.50")) self.assertEquals(stockrecord.num_in_stock, 3) self.assertEquals(prd.categories.count(), 2) - + self.assertEquals(prd.images.count(), 4) + def test_idempotent(self): + product_class = ProductClass.objects.create( + name="Klaas", slug="klaas", requires_shipping=True, track_stock=True + ) + ProductAttribute.objects.create( + name="Henk", + code="henk", + type=ProductAttribute.TEXT, + product_class=product_class, + ) + ProductAttribute.objects.create( + name="Harrie", + code="harrie", + type=ProductAttribute.INTEGER, + product_class=product_class, + ) + + product_class = ProductClassResource(slug="klaas", name="Klaas") + + partner = Partner.objects.create(name="klaas") + + Category.add_root(name="Hatsie", slug="batsie", is_public=True, code="1") + Category.add_root(name="henk", slug="klaas", is_public=True, code="2") + + product_resource = ProductResource( + upc="1234323-2", + title="asdf2", + slug="asdf-asdfasdf2", + description="description", + structure=Product.STANDALONE, + is_discountable=True, + price=D("20"), + availability=2, + currency="EUR", + partner=partner, + product_class=product_class, + images=[ + ImageResource( + caption="gekke caption", + display_order=0, + original=File(self.image, name="harrie.jpg"), + code="12", + ), + ImageResource( + caption="gekke caption 2", + display_order=1, + original=File(self.image, name="vats.jpg"), + code="13", + ), + ], + categories=[CategoryResource(code="2")], + attributes={"henk": "Klaas", "harrie": 1}, + ) + + prd = products_to_db(product_resource) + + prd = Product.objects.get(upc="1234323-2") + + self.assertEquals(prd.title, "asdf2") + self.assertEquals(prd.images.count(), 2) + self.assertEquals(Category.objects.count(), 2) + self.assertEquals(prd.categories.count(), 1) + + self.assertEquals(prd.stockrecords.count(), 1) + stockrecord = prd.stockrecords.first() + self.assertEquals(stockrecord.price, D("20")) + self.assertEquals(stockrecord.num_in_stock, 2) + + self.assertEquals(prd.attr.henk, "Klaas") + self.assertEquals(prd.attr.harrie, 1) + + self.assertEquals(prd.images.count(), 2) + + product_resource = ProductResource( + upc="1234323-2", + title="asdf2", + slug="asdf-asdfasdf2", + description="description", + structure=Product.STANDALONE, + is_discountable=True, + price=D("20"), + availability=2, + currency="EUR", + partner=partner, + product_class=product_class, + images=[ + ImageResource( + caption="gekke caption", + display_order=0, + original=File(self.image, name="harrie.jpg"), + code="12", + ), + ImageResource( + caption="gekke caption 2", + display_order=1, + original=File(self.image, name="vats.jpg"), + code="13", + ), + ], + categories=[CategoryResource(code="2")], + attributes={"henk": "Klaas", "harrie": 1}, + ) + + products_to_db(product_resource) + + prd = Product.objects.get(upc="1234323-2") + + self.assertEquals(prd.title, "asdf2") + self.assertEquals(prd.images.count(), 2) + self.assertEquals(Category.objects.count(), 2) + self.assertEquals(prd.categories.count(), 1) + + self.assertEquals(prd.stockrecords.count(), 1) + stockrecord = prd.stockrecords.first() + self.assertEquals(stockrecord.price, D("20")) + self.assertEquals(stockrecord.num_in_stock, 2) + + self.assertEquals(prd.attr.henk, "Klaas") + self.assertEquals(prd.attr.harrie, 1) + + self.assertEquals(prd.images.count(), 2) + class MultipleProductReverseTest(TestCase): @property @@ -130,16 +281,14 @@ def image(self): output = io.BytesIO() img.save(output, "jpeg") return output - + def test_create_simple_product(self): product_class = ProductClass.objects.create( name="Klaas", slug="klaas", requires_shipping=True, track_stock=True ) Product.objects.create(upc="1234323asd", title="") - product_class = ProductClassResource( - slug="klaas", name="Klaas" - ) - + product_class = ProductClassResource(slug="klaas", name="Klaas") + product_resources = [ ProductResource( upc="1234323asd", @@ -158,7 +307,7 @@ def test_create_simple_product(self): structure=Product.STANDALONE, is_discountable=True, product_class=product_class, - ) + ), ] prd = products_to_db(product_resources) @@ -170,16 +319,20 @@ def test_create_product_with_related_fields(self): name="Klaas", slug="klaas", requires_shipping=True, track_stock=True ) ProductAttribute.objects.create( - name="Henk", code="henk", type=ProductAttribute.TEXT, product_class=product_class + name="Henk", + code="henk", + type=ProductAttribute.TEXT, + product_class=product_class, ) ProductAttribute.objects.create( - name="Harrie", code="harrie", type=ProductAttribute.INTEGER, product_class=product_class - ) - - product_class = ProductClassResource( - slug="klaas", name="Klaas" + name="Harrie", + code="harrie", + type=ProductAttribute.INTEGER, + product_class=product_class, ) + product_class = ProductClassResource(slug="klaas", name="Klaas") + product_resources = [ ProductResource( upc="1234323", @@ -193,10 +346,18 @@ def test_create_product_with_related_fields(self): currency="EUR", product_class=product_class, images=[ - ImageResource(caption="gekke caption", display_order=0, original=File(self.image, name="klaas.jpg")), - ImageResource(caption="gekke caption 2", display_order=1, original=File(self.image, name="harrie.jpg")), + ImageResource( + caption="gekke caption", + display_order=0, + original=File(self.image, name="klaas.jpg"), + ), + ImageResource( + caption="gekke caption 2", + display_order=1, + original=File(self.image, name="harrie.jpg"), + ), ], - attributes={"henk": "Poep", "harrie": 22} + attributes={"henk": "Poep", "harrie": 22}, ), ProductResource( upc="1234323-2", @@ -211,11 +372,19 @@ def test_create_product_with_related_fields(self): partner=Partner.objects.create(name="klaas"), product_class=product_class, images=[ - ImageResource(caption="gekke caption", display_order=0, original=File(self.image, name="klaas.jpg")), - ImageResource(caption="gekke caption 2", display_order=1, original=File(self.image, name="harrie.jpg")), + ImageResource( + caption="gekke caption", + display_order=0, + original=File(self.image, name="klaas.jpg"), + ), + ImageResource( + caption="gekke caption 2", + display_order=1, + original=File(self.image, name="harrie.jpg"), + ), ], - attributes={"henk": "Klaas", "harrie": 1} - ) + attributes={"henk": "Klaas", "harrie": 1}, + ), ] products_to_db(product_resources) @@ -235,4 +404,3 @@ def test_create_product_with_related_fields(self): self.assertEqual(prd2.attr.henk, "Klaas") self.assertEqual(prd2.attr.harrie, 1) - diff --git a/tests/test_settings.py b/tests/test_settings.py index 07ebcba..39977cb 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2,8 +2,7 @@ import oscar # Path helper -location = lambda x: os.path.join( - os.path.dirname(os.path.realpath(__file__)), x) +location = lambda x: os.path.join(os.path.dirname(os.path.realpath(__file__)), x) DEBUG = True @@ -21,32 +20,31 @@ TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - location('templates'), + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [ + location("templates"), ], - 'OPTIONS': { - 'loaders': [ - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', + "OPTIONS": { + "loaders": [ + "django.template.loaders.filesystem.Loader", + "django.template.loaders.app_directories.Loader", ], - 'context_processors': [ - 'django.contrib.auth.context_processors.auth', - 'django.template.context_processors.request', - 'django.template.context_processors.debug', - 'django.template.context_processors.i18n', - 'django.template.context_processors.media', - 'django.template.context_processors.static', - 'django.contrib.messages.context_processors.messages', - + "context_processors": [ + "django.contrib.auth.context_processors.auth", + "django.template.context_processors.request", + "django.template.context_processors.debug", + "django.template.context_processors.i18n", + "django.template.context_processors.media", + "django.template.context_processors.static", + "django.contrib.messages.context_processors.messages", # Oscar specific - 'oscar.apps.search.context_processors.search_form', - 'oscar.apps.communication.notifications.context_processors.notifications', - 'oscar.apps.checkout.context_processors.checkout', - 'oscar.core.context_processors.metadata', + "oscar.apps.search.context_processors.search_form", + "oscar.apps.communication.notifications.context_processors.notifications", + "oscar.apps.checkout.context_processors.checkout", + "oscar.core.context_processors.metadata", ], - 'debug': DEBUG, - } + "debug": DEBUG, + }, } ] @@ -59,13 +57,11 @@ "django.contrib.flatpages", "django.contrib.admin", "django.contrib.messages", - 'widget_tweaks', - 'haystack', - 'treebeard', - 'sorl.thumbnail', - 'django_tables2', - - + "widget_tweaks", + "haystack", + "treebeard", + "sorl.thumbnail", + "django_tables2", # Oscar apps "oscar.config.Shop", "oscar.apps.analytics.apps.AnalyticsConfig", @@ -102,22 +98,19 @@ ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware', - + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.contrib.flatpages.middleware.FlatpageFallbackMiddleware", # Allow languages to be selected - 'django.middleware.locale.LocaleMiddleware', - 'django.middleware.http.ConditionalGetMiddleware', - 'django.middleware.common.CommonMiddleware', - + "django.middleware.locale.LocaleMiddleware", + "django.middleware.http.ConditionalGetMiddleware", + "django.middleware.common.CommonMiddleware", # Ensure a valid basket is added to the request instance for every request - 'oscar.apps.basket.middleware.BasketMiddleware', + "oscar.apps.basket.middleware.BasketMiddleware", ] from oscar.defaults import * @@ -132,4 +125,4 @@ STATIC_URL = "/static/" -ROOT_URLCONF = "urls" \ No newline at end of file +ROOT_URLCONF = "urls" From 12648fe9f8239ddb1d014f7309734bd5c6f10662 Mon Sep 17 00:00:00 2001 From: Viggo de Vries Date: Fri, 15 Dec 2023 11:28:05 +0100 Subject: [PATCH 4/6] Make it easier to change the identifiers mapping --- oscar_odin/mappings/catalogue.py | 6 +++-- oscar_odin/mappings/constants.py | 16 +++++++++++++ oscar_odin/mappings/context.py | 40 ++++++++------------------------ oscar_odin/mappings/utils.py | 2 +- 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/oscar_odin/mappings/catalogue.py b/oscar_odin/mappings/catalogue.py index 6ffa58b..b8f6366 100644 --- a/oscar_odin/mappings/catalogue.py +++ b/oscar_odin/mappings/catalogue.py @@ -26,7 +26,7 @@ save_foreign_keys, ) from .context import ModelMapperContext -from .constants import ALL_CATALOGUE_FIELDS +from .constants import ALL_CATALOGUE_FIELDS, MODEL_IDENTIFIERS_MAPPING __all__ = ( "ProductImageToResource", @@ -393,6 +393,7 @@ def products_to_db( products: List[resources.catalogue.Product], rollback=True, fields_to_update=ALL_CATALOGUE_FIELDS, + identifier_mapping=MODEL_IDENTIFIERS_MAPPING, ) -> Tuple[List[ProductModel], Dict]: """Map mulitple products to a model and store them in the database. @@ -401,7 +402,8 @@ def products_to_db( At last all related models like images, stockrecords, and related_products can will be saved and set on the product. """ instances, context = products_to_model(products) - context.add_fields_to_update(fields_to_update) + context.fields_to_update = fields_to_update + context.identifier_mapping = identifier_mapping errors = {} diff --git a/oscar_odin/mappings/constants.py b/oscar_odin/mappings/constants.py index f207bf9..b6fa471 100644 --- a/oscar_odin/mappings/constants.py +++ b/oscar_odin/mappings/constants.py @@ -1,3 +1,11 @@ +from oscar.core.loading import get_model + +Product = get_model("catalogue", "Product") +Category = get_model("catalogue", "Category") +StockRecord = get_model("partner", "StockRecord") +ProductClass = get_model("catalogue", "ProductClass") +ProductImage = get_model("catalogue", "ProductImage") + PRODUCT_STRUCTURE = "Product.structure" PRODUCT_IS_PUBLIC = "Product.is_public" PRODUCT_UPC = "Product.upc" @@ -76,3 +84,11 @@ ALL_CATALOGUE_FIELDS = ( ALL_PRODUCT_FIELDS + ALL_PRODUCTIMAGE_FIELDS + ALL_STOCKRECORD_FIELDS ) + +MODEL_IDENTIFIERS_MAPPING = { + Category: ("code",), + Product: ("upc",), + StockRecord: ("product_id",), + ProductClass: ("slug",), + ProductImage: ("code",), +} diff --git a/oscar_odin/mappings/context.py b/oscar_odin/mappings/context.py index ae564aa..4138d1e 100644 --- a/oscar_odin/mappings/context.py +++ b/oscar_odin/mappings/context.py @@ -14,33 +14,11 @@ ProductImage = get_model("catalogue", "ProductImage") -MODEL_IDENTIFIERS_MAPPING = { - Category: ("code",), - Product: ("upc",), - StockRecord: ("product_id",), - ProductClass: ("slug",), - ProductImage: ("code",), -} - - -def get_unique_id_list(Model, instances): - unique_id_list = [] - identifiers = MODEL_IDENTIFIERS_MAPPING.get(Model, {}) - - if identifiers: - for instance in instances: - unique_id_list.append( - [getattr(instance, identifier) for identifier in identifiers] - ) - - return unique_id_list, identifiers - - -def get_instances_to_create_or_update(Model, instances): +def get_instances_to_create_or_update(Model, instances, identifier_mapping): instances_to_create = [] instances_to_update = [] - unique_id_list, identifiers = get_unique_id_list(Model, instances) + identifiers = identifier_mapping.get(Model, {}) if identifiers: id_mapping = in_bulk( @@ -72,8 +50,8 @@ class ModelMapperContext(dict): many_to_many_items = None many_to_one_items = None one_to_many_items = None - source_fields = None attribute_data = None + identifier_mapping = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -82,6 +60,7 @@ def __init__(self, *args, **kwargs): self.many_to_one_items = defaultdict(list) self.one_to_many_items = defaultdict(list) self.fields_to_update = defaultdict(list) + self.identifier_mapping = defaultdict(tuple) self.attribute_data = [] def __bool__(self): @@ -103,9 +82,6 @@ def add_instance_to_fk_items(self, field, instance): if not instance.pk: self.foreign_key_items[field] += [instance] - def add_fields_to_update(self, fields_to_update): - self.fields_to_update = fields_to_update - def get_fields_to_update(self, Model): modelname = "%s." % Model.__name__ return [ @@ -127,7 +103,9 @@ def get_create_and_update_relations(self, related_instance_items): ( instances_to_create, instances_to_update, - ) = get_instances_to_create_or_update(relation.related_model, all_instances) + ) = get_instances_to_create_or_update( + relation.related_model, all_instances, self.identifier_mapping + ) to_create[relation].extend(instances_to_create) to_update[relation].extend(instances_to_update) @@ -152,7 +130,9 @@ def get_fk_relations(self): ( instances_to_create, instances_to_update, - ) = get_instances_to_create_or_update(relation.related_model, instances) + ) = get_instances_to_create_or_update( + relation.related_model, instances, self.identifier_mapping + ) to_create[relation].extend(instances_to_create) to_update[relation].extend(instances_to_update) diff --git a/oscar_odin/mappings/utils.py b/oscar_odin/mappings/utils.py index 786f220..7c484e0 100644 --- a/oscar_odin/mappings/utils.py +++ b/oscar_odin/mappings/utils.py @@ -37,7 +37,7 @@ def save_foreign_keys(context, errors): def save_products(instances, context, errors): instances_to_create, instances_to_update = get_instances_to_create_or_update( - Product, instances + Product, instances, context.identifier_mapping ) validated_create_instances, errors = validate_instances(instances_to_create, errors) From 6b0796378382d2ebeb016d11942a2e6f8584a583 Mon Sep 17 00:00:00 2001 From: Viggo de Vries Date: Fri, 15 Dec 2023 14:57:11 +0100 Subject: [PATCH 5/6] Add reallife testcase --- Makefile | 2 +- .../fixtures/oscar_odin/csv/products.csv | 60 ++++++ oscar_odin/mappings/context.py | 8 - oscar_odin/resources/catalogue.py | 1 + tests/reverse/test_reallifecase.py | 203 ++++++++++++++++++ 5 files changed, 265 insertions(+), 9 deletions(-) create mode 100644 oscar_odin/fixtures/oscar_odin/csv/products.csv create mode 100644 tests/reverse/test_reallifecase.py diff --git a/Makefile b/Makefile index a67d517..88d570e 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ lint: fail-if-no-virtualenv pylint oscar_odin/ test: fail-if-no-virtualenv - python3 runtests.py test tests/ + python3 runtests.py test tests.reverse.test_reallifecase black: @black oscar_odin/ diff --git a/oscar_odin/fixtures/oscar_odin/csv/products.csv b/oscar_odin/fixtures/oscar_odin/csv/products.csv new file mode 100644 index 0000000..531f282 --- /dev/null +++ b/oscar_odin/fixtures/oscar_odin/csv/products.csv @@ -0,0 +1,60 @@ +id,name,category_id,weight,weight_type,price,weight_price,image,app_image,description,EAN,number,VAT,supplier_id,active,anonymous,quantity,unit,admin_approved,is_new_introduction,tags,brand_id,stock_manager,is_deleted,is_weightedproduct,storage_type,psinfood_id,package_ean,warehouse_id,assortmentclass,package,package_quantity,package_unit,Supplier,WareHouse,ProductInvitations +19860,RAW CAKES VANILLE,101,1,kilo,41.85,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1698660334306_449650113_e2%20(Middel).jpg?alt=media&token=f51d26f5-18f6-46b5-bd12-a68d4a15fcc0,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1698660334306_449650113_e2%20(Middel).jpg?alt=media&token=f51d26f5-18f6-46b5-bd12-a68d4a15fcc0,,5407005642115,,9,1049,true,,,doos,true,,,,,,,frozen,,,2,franchise,,,,[object Object],[object Object], +19823,MACARON PEACH-MANGO 50ST,213,1,kilo,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1696026421065_825167321_PEACH%20MANGO.png?alt=media&token=0a318299-bfe6-425b-83c5-4516eeee7cac,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1696026421065_825167321_PEACH%20MANGO.png?alt=media&token=0a318299-bfe6-425b-83c5-4516eeee7cac,,2100140000002,111111,9,1052,true,,,doos,true,,,,,,,frozen,,,2,normal,,,,[object Object],[object Object],[object Object] +19806,White Chocolate Tray Brownie Pre-Cut(6x6),213,1,kilo,28.11,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695375582459_129911926_t%2010.57.14.jpeg?alt=media&token=9d22dd54-48eb-403c-b006-8311eb5bbe6f,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695375582459_129911926_t%2010.57.14.jpeg?alt=media&token=9d22dd54-48eb-403c-b006-8311eb5bbe6f,,8683880757695,BR9021,9,1053,true,,,stuk,true,,,,,,,frozen,,,2,franchise,,,,[object Object],[object Object], +19805,Hazelnut Tray Brownie Pre-Cut(6x6),213,1,kilo,24.64,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695370950969_911725953_.10.52%20(5).jpeg?alt=media&token=d35cd705-9242-484d-b900-151530166457,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695370950969_911725953_.10.52%20(5).jpeg?alt=media&token=d35cd705-9242-484d-b900-151530166457,,8683880757657,BRTP0008,9,1053,true,,,stuk,true,,,,,,,frozen,,,2,franchise,,,,[object Object],[object Object], +19804,Orange Tray Brownie Pre-Cut(6x6),213,1,kilo,21.45,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695370852426_965127482_.10.51%20(1).jpeg?alt=media&token=a2a16af9-1e0f-4f23-b892-adc5660dfd4b,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695370852426_965127482_.10.51%20(1).jpeg?alt=media&token=a2a16af9-1e0f-4f23-b892-adc5660dfd4b,,8683880757558,BRTP0004,9,1053,true,,,stuk,true,,,,,,,frozen,,,2,franchise,,,,[object Object],[object Object], +19803,Lemon Tray Brownie Pre-Cut(6x6),213,1,kilo,21.45,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695370710042_594397468_.16.35%20(1).jpeg?alt=media&token=956723d3-bf17-443a-bf00-8d8509ace496,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695370710042_594397468_.16.35%20(1).jpeg?alt=media&token=956723d3-bf17-443a-bf00-8d8509ace496,,8683880757619,BRTP0013,9,1053,true,,,stuk,true,,,,,,,frozen,,,2,franchise,,,,[object Object],[object Object], +19802,Classic Tray Brownie Pre-Cut(6x6),213,1,kilo,19.91,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695370573220_373873097_.10.52%20(8).jpeg?alt=media&token=192bbbb5-ac35-4405-85c9-882b0cedc1c7,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695370573220_373873097_.10.52%20(8).jpeg?alt=media&token=192bbbb5-ac35-4405-85c9-882b0cedc1c7,,8683752709159,BR9026,9,1053,true,,,stuk,true,,,,,,,frozen,,,2,franchise,,,,[object Object],[object Object], +19801,Peanut Butter Tray Brownie Pre-Cut(6x6),213,1,kilo,24.64,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695370198571_331792047_.10.52%20(1).jpeg?alt=media&token=4c09970c-724d-4d35-b9ba-75d6d2260078,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1695370198571_331792047_.10.52%20(1).jpeg?alt=media&token=4c09970c-724d-4d35-b9ba-75d6d2260078,,8683880757534,BR9020,9,1053,true,,,stuk,true,,,,,,,frozen,,,2,franchise,,,,[object Object],[object Object], +18835,MACARON VANILLE 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642229370_393196562_Vanilla.jpg?alt=media&token=c52947a4-f9fa-42a6-ae1b-3751d1e70f90,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642229370_393196562_Vanilla.jpg?alt=media&token=c52947a4-f9fa-42a6-ae1b-3751d1e70f90,,2100030000006,391735,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18834,MACARON TIRAMISU 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642221134_852498327_Tiramisu.jpg?alt=media&token=25ba641d-b735-4f0c-8918-8efe2c3c5c7e,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642221134_852498327_Tiramisu.jpg?alt=media&token=25ba641d-b735-4f0c-8918-8efe2c3c5c7e,,2100200000003,391794,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18833,MACARON STRAWBERRY 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642212528_73755141_Strawberry.jpg?alt=media&token=65fe8350-e2b3-410b-afe4-bf4c3e79968a,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642212528_73755141_Strawberry.jpg?alt=media&token=65fe8350-e2b3-410b-afe4-bf4c3e79968a,,2100120000008,391875,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18832,MACARON SPECULOOS 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642202084_244631676_Speculoos.jpg?alt=media&token=13e85485-48ed-40fd-95ee-cac8c2f0a71d,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642202084_244631676_Speculoos.jpg?alt=media&token=13e85485-48ed-40fd-95ee-cac8c2f0a71d,,2100210000000,391840,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18831,MACARON REDVELVET 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642190061_570396783_Red%20Velvet.jpg?alt=media&token=bae77b87-d6ac-4a88-aa97-a24c96b94f4c,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642190061_570396783_Red%20Velvet.jpg?alt=media&token=bae77b87-d6ac-4a88-aa97-a24c96b94f4c,,2100180000000,391743,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18830,MACARON RASPBERRY 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642180800_193302737_Raspberry.jpg?alt=media&token=1d1c2463-e267-44f0-85be-f1d1011bff47,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642180800_193302737_Raspberry.jpg?alt=media&token=1d1c2463-e267-44f0-85be-f1d1011bff47,,2100170000003,391891,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18829,MACARON PISTACHIO 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642169437_570501660_Pistachio%20Nuts.jpg?alt=media&token=97b60c04-7c25-4363-bcaa-340f21f1f538,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642169437_570501660_Pistachio%20Nuts.jpg?alt=media&token=97b60c04-7c25-4363-bcaa-340f21f1f538,,2100040000003,391808,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18828,MACARON PEANUT BUTTER 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642245865_516395100_Peanut%20Buttercup.jpg?alt=media&token=e840b802-3b43-49a3-83cf-e78cc67562b0,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642245865_516395100_Peanut%20Buttercup.jpg?alt=media&token=e840b802-3b43-49a3-83cf-e78cc67562b0,,2100220000007,391883,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18827,MACARON PASSION FRUIT 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642155707_910558237_Passion%20Fruit.jpg?alt=media&token=a1fb1def-b769-4f54-97db-e125c0a2fbe9,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642155707_910558237_Passion%20Fruit.jpg?alt=media&token=a1fb1def-b769-4f54-97db-e125c0a2fbe9,,2100160000006,391956,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18826,MACARON OREO 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642062663_588874497_eo%20High%20Res.jpg?alt=media&token=96d29888-e8c9-4957-a407-e3b2adb7c95e,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642062663_588874497_eo%20High%20Res.jpg?alt=media&token=96d29888-e8c9-4957-a407-e3b2adb7c95e,,2100240000001,399981,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18825,MACARON LEMON 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642050668_449052725_Lemon.jpg?alt=media&token=1ce2d574-ae7c-4161-a443-25a4bdcd457c,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642050668_449052725_Lemon.jpg?alt=media&token=1ce2d574-ae7c-4161-a443-25a4bdcd457c,,2100110000001,391867,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18824,MACARON ITALIAN COOKIE 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642039711_303946959_Italian%20Cookie.jpg?alt=media&token=f3c9dcc2-f363-4289-ae59-8ebd7f7124c4,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642039711_303946959_Italian%20Cookie.jpg?alt=media&token=f3c9dcc2-f363-4289-ae59-8ebd7f7124c4,,2100190000007,391905,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18823,MACARON HAZELNUT NUTELLA 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642002722_466408713_ut%20%20Nutella.jpg?alt=media&token=b807df54-86f0-44df-a739-e7db386cf0cb,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688642002722_466408713_ut%20%20Nutella.jpg?alt=media&token=b807df54-86f0-44df-a739-e7db386cf0cb,,2100070000004,391824,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18822,MACARON FROZEN YOGHURT 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641988355_59300671_Frozen%20Yoghurt.jpg?alt=media&token=b56a6ec7-bfb4-44ea-82c0-216a2d75d8c8,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641988355_59300671_Frozen%20Yoghurt.jpg?alt=media&token=b56a6ec7-bfb4-44ea-82c0-216a2d75d8c8,,2100090000008,391859,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18821,MACARON MOJITO 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641979086_125362894_Fresh%20Mint.jpg?alt=media&token=254fc275-776c-4bfa-b76a-df54ac1fae59,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641979086_125362894_Fresh%20Mint.jpg?alt=media&token=254fc275-776c-4bfa-b76a-df54ac1fae59,,2100140000002,391972,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18820,MACARON DUTCH COOKIE 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641969573_715256835_kie%20%26%20Cream.jpg?alt=media&token=d65e3d7e-bcbd-450b-8001-3454dbcde120,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641969573_715256835_kie%20%26%20Cream.jpg?alt=media&token=d65e3d7e-bcbd-450b-8001-3454dbcde120,,2100080000001,391786,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18819,MACARON COFFEE/CARAMEL MACCHIATO 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641955426_899604401_Coffee.jpg?alt=media&token=b8dceb37-9229-4c80-bd03-ac357330d6cd,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641955426_899604401_Coffee.jpg?alt=media&token=b8dceb37-9229-4c80-bd03-ac357330d6cd,,2100050000000,391913,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18817,MACARON CHOCOLATE 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641942312_728997024_Chocolate.jpg?alt=media&token=ef53bb26-d1f3-4ebd-a4dd-30df5fc31261,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641942312_728997024_Chocolate.jpg?alt=media&token=ef53bb26-d1f3-4ebd-a4dd-30df5fc31261,,2100010000002,391832,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18816,MACARON CHEESECAKE 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641931902_175491984_%20Cheesecake.jpg?alt=media&token=4b4d9b3e-cdc1-4a83-9a98-6ca2190feb6b,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641931902_175491984_%20Cheesecake.jpg?alt=media&token=4b4d9b3e-cdc1-4a83-9a98-6ca2190feb6b,,2100020000009,391778,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18815,MACARON CARAMEL SALTED 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641919722_526344378_Salted%20Caramel.jpg?alt=media&token=ca6b744f-e2e8-4823-9970-11f0aa4862b0,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641919722_526344378_Salted%20Caramel.jpg?alt=media&token=ca6b744f-e2e8-4823-9970-11f0aa4862b0,,2100230000004,391751,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18814,MACARON BUBBLEGUM 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641906499_813456162_Bubble%20Gum.jpg?alt=media&token=f0b8ade1-1e4f-4ee7-a4ee-adfbbb0cb188,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641906499_813456162_Bubble%20Gum.jpg?alt=media&token=f0b8ade1-1e4f-4ee7-a4ee-adfbbb0cb188,,2100130000005,391921,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18813,MACARON BLACK CHERRY PIE 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641892658_372017613_Black%20Cherry%20Pie.jpg?alt=media&token=3f80e00a-5700-45f0-961d-da2b14f129d6,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641892658_372017613_Black%20Cherry%20Pie.jpg?alt=media&token=3f80e00a-5700-45f0-961d-da2b14f129d6,,2100150000009,391816,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18812,MACARON BANANA 50ST,213,1,doos,49.5,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641879136_856647310_Banana.jpg?alt=media&token=ec620157-c8bd-4d34-aec2-389db2bfae6d,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688641879136_856647310_Banana.jpg?alt=media&token=ec620157-c8bd-4d34-aec2-389db2bfae6d,,2100100000004,391948,9,1052,true,,,doos,true,,,,0,,,frozen,,NaN,2,normal,,,,[object Object],[object Object],[object Object] +18809,MACARON TASJES MED 250ST,264,1,kilo,97.5,,,,,8716739157190,53333,9,1052,true,,,doos,true,,,,,,,ambient,,,2,normal,,,,[object Object],[object Object],[object Object] +18808,MACARON 6-PACK DHM DOOSJE,264,1,kilo,81.9,,,,,8716739157176,53317,9,1052,true,,,doos,true,,,,,,,ambient,,,2,normal,,,,[object Object],[object Object],[object Object] +18807,MACARON 6-PACK DHM INLAY,264,1,kilo,35.1,,,,,8716739157169,53309,9,1052,true,,,doos,true,,,,,,,ambient,,,2,normal,,,,[object Object],[object Object],[object Object] +18806,MACARON 9-PACK DHM DOOSJE,264,1,kilo,67.2,,,,,8716739157152,53295,9,1052,true,,,doos,true,,,,,,,ambient,,,2,normal,,,,[object Object],[object Object],[object Object] +18805,MACARON 9-PACK DHM INLAY,264,1,kilo,28.8,,,,,8716739157145,53287,9,1052,true,,,doos,true,,,,,,,ambient,,,2,normal,,,,[object Object],[object Object],[object Object] +18804,MACARON 12-PACK DHM DOOSJE,264,1,kilo,40.25,,,,,8716739157138,53279,9,1052,true,,,doos,true,,,,,,,ambient,,,2,normal,,,,[object Object],[object Object],[object Object] +18803,MACARON 12-PACK DHM INLAY,264,1,kilo,17,,,,,8716739157121,53260,9,1052,true,,,stuk,true,,,,,,,ambient,,,2,normal,,,,[object Object],[object Object],[object Object] +18801,SUMMERBERRY STACK,101,1,kilo,121.39,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688545727479_916442946_y%20stack%208st.png?alt=media&token=673f6972-f9ad-4f93-8c70-343eb64ec0ab,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688545727479_916442946_y%20stack%208st.png?alt=media&token=673f6972-f9ad-4f93-8c70-343eb64ec0ab,"
Sweet Street, 8x 8st 

",10749017016662,442933,9,1049,true,,,doos,true,,Sweet Street,,,,,frozen,,,2,franchise,,,,[object Object],[object Object], +18742,VEGAN GEBAKJE AARDBEIEN,101,1,doos,17.68,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688391351283_614674814_e%20aardbeien.png?alt=media&token=0f420823-fc76-4dec-b2f0-003779d1f7db,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688391351283_614674814_e%20aardbeien.png?alt=media&token=0f420823-fc76-4dec-b2f0-003779d1f7db,"
Patisserie Unique 12 st
",4004311195239,384291,9,1049,true,,,doos,true,,Patisserie Unique,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18741,SKY HIGH APPLE PIE 12P,101,1,doos,62.65,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688391285148_827181427_h%20Apple%20pie.png?alt=media&token=a9f5cb03-3513-4fb7-b7d3-b4206be49459,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688391285148_827181427_h%20Apple%20pie.png?alt=media&token=a9f5cb03-3513-4fb7-b7d3-b4206be49459,"
Cut The Cake 4 x 12 p
",8715196556096,449776,9,1049,true,,,doos,true,,Cut The Cake,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18740,REBELLIOUS RED VELVET 14P,101,1,doos,71.17,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688391233519_79422497_%20Red%20Velvet.png?alt=media&token=1e1f6414-dc3c-4fe5-8ab3-752d0a51e94b,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688391233519_79422497_%20Red%20Velvet.png?alt=media&token=1e1f6414-dc3c-4fe5-8ab3-752d0a51e94b,"
Cut The Cake 4 x 14 p
",8715196556010,449768,9,1049,true,,,doos,true,,Cut The Cake,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18739,RAW CAKES TIRAMISU,101,1,doos,41.32,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688391136178_655980627_440914%20Tiramisu.jpg?alt=media&token=eb8736ad-5f05-4382-84cc-afe42900325a,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688391136178_655980627_440914%20Tiramisu.jpg?alt=media&token=eb8736ad-5f05-4382-84cc-afe42900325a,"
Nats Rawline 3 x 12 st
",5407005640586,440914,9,1049,true,,,doos,true,,Nats Rawline,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18738,RAW CAKES RASPBERRY LIME,101,1,doos,42.26,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688391083360_848572299_erry%20%26%20Lime.jpg?alt=media&token=c7105ed3-786a-4016-b8f0-5d56c1d9fa00,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688391083360_848572299_erry%20%26%20Lime.jpg?alt=media&token=c7105ed3-786a-4016-b8f0-5d56c1d9fa00,"
Nats Rawline 3 x 12 st
",5407005640876,443719,9,1049,true,,,doos,true,,Nats Rawline,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18737,RAW CAKES RASPBERRY BLUEBERRY,101,1,doos,41.85,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688390999106_179293923_y-blueberry.jpg?alt=media&token=91bb334e-9e79-486f-85c4-7e08e23e8bc7,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688390999106_179293923_y-blueberry.jpg?alt=media&token=91bb334e-9e79-486f-85c4-7e08e23e8bc7,"
Nats Rawline 3 x 12 st
",5407005640470,440930,9,1049,true,,,doos,true,,Nats Rawline,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18735,RAW CAKES LIME MANGO,101,1,doos,41.85,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688390856719_321727338_ime%20%E2%80%93%20Mango.jpg?alt=media&token=cfa3f986-1ef4-4309-9bb3-2c84bc7dddf2,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688390856719_321727338_ime%20%E2%80%93%20Mango.jpg?alt=media&token=cfa3f986-1ef4-4309-9bb3-2c84bc7dddf2,"
Nats Rawline 3 x 12 st
",5407005640579,440949,9,1049,true,,,doos,true,,Nats Rawline,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18734,RAW CAKES CHOCOLATE HAZELNUT,101,1,doos,41.85,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688390769219_884740623_ts%20raw%20cake.jpg?alt=media&token=a0e6dfde-791b-4177-9149-309b9a4919b9,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688390769219_884740623_ts%20raw%20cake.jpg?alt=media&token=a0e6dfde-791b-4177-9149-309b9a4919b9,"
Nats Rawline 3 x 12 st
",5407005641125,443700,9,1049,true,,,doos,true,,Nats Rawline,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18733,RAW CAKES CARAMEL CHOCOLATE,101,1,doos,41.32,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688390523791_316569728_%20chocolate1.jpg?alt=media&token=5018d156-1db1-4c79-a3e4-31c431f51659,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688390523791_316569728_%20chocolate1.jpg?alt=media&token=5018d156-1db1-4c79-a3e4-31c431f51659,"
Nats Rawline 3 x 12 st
",5407005640593,440922,9,1049,true,,,doos,true,,Nats Rawline,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18732,RASPBERRY WHITE CHOCOLATE CC,101,1,doos,80.96,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688390243381_375351179_%20Cheesecake.png?alt=media&token=24d91073-d782-4d6d-a60d-c598ea170cb6,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688390243381_375351179_%20Cheesecake.png?alt=media&token=24d91073-d782-4d6d-a60d-c598ea170cb6,"
Sweet Street 2 x 14 p
",10749017012558,374784,9,1049,true,,,doos,true,,Sweet Street,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18731,PASSIONATE POMEGRANATE 14P,101,1,doos,70.39,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389760672_248051217_Pomegranate.png?alt=media&token=3f7d38cb-aa70-48f8-a521-fdee15174da0,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389760672_248051217_Pomegranate.png?alt=media&token=3f7d38cb-aa70-48f8-a521-fdee15174da0,"
Cut The Cake 4 x 14 p
",8715196556119,449822,9,1049,true,,,doos,true,,Cut The Cake,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18730,NEW YORK CHEESECAKE,101,1,doos,94.06,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389707108_901140884_%20Cheesecake.png?alt=media&token=8c6a3335-1a7a-4981-9549-debf85763290,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389707108_901140884_%20Cheesecake.png?alt=media&token=8c6a3335-1a7a-4981-9549-debf85763290,"
Sweet Street 4 x 16 p
",10749017009145,317071,9,1049,true,,,doos,true,,Sweet Street,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18729,HAZELINO KARAMEL,101,1,doos,21.26,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389520355_211854468_ino%20Karamel.png?alt=media&token=93185c28-5deb-430e-bf04-b6c6e8a09c99,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389520355_211854468_ino%20Karamel.png?alt=media&token=93185c28-5deb-430e-bf04-b6c6e8a09c99,"
Patisserie Unique 16 st
",8718026245085,398977,9,1049,true,,,doos,true,,Patisserie Unique,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18728,HAZELINO CHRISTOFFEL,101,1,doos,21.26,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389409084_706530979_Christoffel.png?alt=media&token=bc174677-d9cc-4e91-9365-41c50293b7f1,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389409084_706530979_Christoffel.png?alt=media&token=bc174677-d9cc-4e91-9365-41c50293b7f1,"
Patisserie Unique 16 st
",8718026245054,398985,9,1049,true,,,doos,true,,Patisserie Unique,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18727,GLUTEN LACTOSEVRIJ CHOCOLADE,101,1,doos,24.12,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389319526_419892598_ade%20gebakje.png?alt=media&token=28253352-1cd2-4df5-be1a-1bea3dc84383,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389319526_419892598_ade%20gebakje.png?alt=media&token=28253352-1cd2-4df5-be1a-1bea3dc84383,"
Patisserie Unique 12 st
",4004311187463,384275,9,1049,true,,,doos,true,,Patisserie Unique,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18726,GLUTEN LACTOSEVRIJ AARDBEIEN,101,1,doos,24.12,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389226227_386471740_ien%20gebakje.png?alt=media&token=7b964119-1bcf-4eaf-be0a-5b545fd1253b,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688389226227_386471740_ien%20gebakje.png?alt=media&token=7b964119-1bcf-4eaf-be0a-5b545fd1253b,"
Patisserie Unique 12 st
",4004311187371,384305,9,1049,true,,,doos,true,,Patisserie Unique,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18725,CREME BRULEE CHEESE,101,1,doos,72.13,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688385861936_343085633_%20Cheesecake.png?alt=media&token=6830c9a5-0a60-46a0-b891-26cfab7f1aa8,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688385861936_343085633_%20Cheesecake.png?alt=media&token=6830c9a5-0a60-46a0-b891-26cfab7f1aa8,"
Sweet Street 2 x 14 p
",10749017012572,316733,9,1049,true,,,doos,true,,Sweet Street,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18724,CRAZY CARROT CAKE 14P,101,1,doos,71.17,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688385320516_292955121_Carrot%20Cake.png?alt=media&token=daca2c37-7eb2-4613-ab8d-47a5f304e517,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688385320516_292955121_Carrot%20Cake.png?alt=media&token=daca2c37-7eb2-4613-ab8d-47a5f304e517,"
Cut The Cake 4 x 14 p
",8715196556034,449806,9,1049,true,,,doos,true,,Cut The Cake,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18722,CHEEKY CHOCOLATE 14P,101,1,doos,70,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688384904957_805496646_y%20Chocolate.png?alt=media&token=60cb9220-43c4-4772-b0ca-647ccde5d8da,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688384904957_805496646_y%20Chocolate.png?alt=media&token=60cb9220-43c4-4772-b0ca-647ccde5d8da,"
Cut The Cake 4 x 14 p
",8715196556058,449792,9,1049,true,,,doos,true,,Cut the Cake,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], +18721,BOMBASTIC BROWNIE 14P,101,1,doos,67.13,,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688384855959_904936215_tic%20Brownie.png?alt=media&token=1fd1eb92-8f6f-472c-8be8-860ebaa686e1,https://firebasestorage.googleapis.com/v0/b/app-verstrade.appspot.com/o/angular%2Fprod%2Fproducts%2F1688384855959_904936215_tic%20Brownie.png?alt=media&token=1fd1eb92-8f6f-472c-8be8-860ebaa686e1,"
Cut The Cake 4 x 14 p
",8715196556133,449784,9,1049,true,,,doos,true,,Cut The Cake,,0,,,frozen,,NaN,2,franchise,,,,[object Object],[object Object], diff --git a/oscar_odin/mappings/context.py b/oscar_odin/mappings/context.py index 4138d1e..988763d 100644 --- a/oscar_odin/mappings/context.py +++ b/oscar_odin/mappings/context.py @@ -5,14 +5,6 @@ from oscar_odin.utils import in_bulk -from oscar.core.loading import get_model - -Product = get_model("catalogue", "Product") -Category = get_model("catalogue", "Category") -StockRecord = get_model("partner", "StockRecord") -ProductClass = get_model("catalogue", "ProductClass") -ProductImage = get_model("catalogue", "ProductImage") - def get_instances_to_create_or_update(Model, instances, identifier_mapping): instances_to_create = [] diff --git a/oscar_odin/resources/catalogue.py b/oscar_odin/resources/catalogue.py index 84c784f..e644219 100644 --- a/oscar_odin/resources/catalogue.py +++ b/oscar_odin/resources/catalogue.py @@ -106,6 +106,7 @@ class Product(OscarCatalogue): images: List[Image] = odin.Options(empty=True) rating: Optional[float] is_discountable: bool + is_public: bool # Price information price: Decimal = DecimalField() diff --git a/tests/reverse/test_reallifecase.py b/tests/reverse/test_reallifecase.py new file mode 100644 index 0000000..805d391 --- /dev/null +++ b/tests/reverse/test_reallifecase.py @@ -0,0 +1,203 @@ +import io +import PIL +import odin +import requests + +from urllib.parse import urlparse + +from odin.codecs import csv_codec +from os import path +from decimal import Decimal as D + +from django.core.files import File +from django.test import TestCase + +from oscar.core.loading import get_model, get_class + +from django.utils.text import slugify + +from oscar_odin.fields import DecimalField +from oscar_odin.mappings.catalogue import products_to_db +from oscar_odin.resources.catalogue import ( + Product as ProductResource, + Image as ImageResource, + ProductClass as ProductClassResource, + Category as CategoryResource, + ProductAttributeValue as ProductAttributeValueResource, +) + +Product = get_model("catalogue", "Product") +ProductClass = get_model("catalogue", "ProductClass") +ProductAttribute = get_model("catalogue", "ProductAttribute") +ProductAttributeValue = get_model("catalogue", "ProductAttributeValue") +ProductImage = get_model("catalogue", "ProductImage") +Category = get_model("catalogue", "Category") +Partner = get_model("partner", "Partner") + +create_from_breadcrumbs = get_class("catalogue.categories", "create_from_breadcrumbs") + + +class CSVProductResource(odin.Resource): + id = odin.IntegerField() + name = odin.StringField() + category_id = odin.IntegerField() + weight = odin.IntegerField() + weight_type = odin.StringField() + price = DecimalField() + image = odin.StringField(null=True) + app_image = odin.StringField(null=True) + description = odin.StringField(null=True) + ean = odin.StringField(null=True) + number = odin.StringField(null=True) + supplier_id = odin.IntegerField() + active = odin.BooleanField() + unit = odin.StringField() + tags = odin.StringField(null=True) + storage_type = odin.StringField() + assortmentclass = odin.StringField() + + +class CSVProductMapping(odin.Mapping): + from_obj = CSVProductResource + to_obj = ProductResource + + mappings = ( + odin.define(from_field="number", to_field="upc"), + odin.define(from_field="name", to_field="title"), + odin.define(from_field="active", to_field="is_public"), + ) + + @odin.map_list_field(from_field="category_id") + def categories(self, category_id): + return [CategoryResource(code=category_id)] + + @odin.map_field(from_field="name") + def slug(self, name): + return slugify(name) + + @odin.map_field( + from_field=["weight", "weight_type", "ean", "unit", "tags", "storage_type"] + ) + def attributes(self, weight, weight_type, ean, unit, tags, storage_type): + return { + "weight": weight, + "weight_type": weight_type, + "ean": ean, + "unit": unit, + "tags": tags, + "storage_type": storage_type, + } + + @odin.map_list_field(from_field=["image", "app_image"]) + def images(self, image, app_image): + images = [] + + if image: + response = requests.get(image) + a = urlparse(image) + img = File(io.BytesIO(response.content), name=path.basename(a.path)) + images.append( + ImageResource(display_order=0, code=image, caption="", original=img) + ) + + if app_image and app_image != image: + response = requests.get(app_image) + a = urlparse(app_image) + img = File(io.BytesIO(response.content), name=path.basename(a.path)) + images.append( + ImageResource(display_order=1, caption="", code=image, original=img) + ) + + return images + + @odin.map_field(from_field="supplier_id") + def partner(self, supplier_id): + partner, _ = Partner.objects.get_or_create(name=supplier_id) + return partner + + @odin.assign_field + def product_class(self): + return ProductClassResource(slug="standard") + + @odin.assign_field + def structure(self): + return Product.STANDALONE + + @odin.assign_field + def is_discountable(self): + return True + + +class RealLifeTest(TestCase): + def test_mapping(self): + # Create product class + product_class, _ = ProductClass.objects.get_or_create( + slug="standard", + defaults={ + "name": "Standard product class", + "requires_shipping": True, + "track_stock": False, + }, + ) + ProductAttribute.objects.get_or_create( + code="weight", + product_class=product_class, + defaults={"name": "Weight", "type": ProductAttribute.INTEGER}, + ) + ProductAttribute.objects.get_or_create( + code="weight_type", + product_class=product_class, + defaults={"name": "Weight type", "type": ProductAttribute.TEXT}, + ) + ProductAttribute.objects.get_or_create( + code="ean", + product_class=product_class, + defaults={"name": "EAN", "type": ProductAttribute.TEXT}, + ) + ProductAttribute.objects.get_or_create( + code="unit", + product_class=product_class, + defaults={"name": "Unit", "type": ProductAttribute.TEXT}, + ) + ProductAttribute.objects.get_or_create( + code="tags", + product_class=product_class, + defaults={"name": "Tags", "type": ProductAttribute.TEXT}, + ) + ProductAttribute.objects.get_or_create( + code="storage_type", + product_class=product_class, + defaults={"name": "Storage type", "type": ProductAttribute.TEXT}, + ) + + # Create all the categories at first and assign a unique code + for cat_id in ["101", "213", "264"]: + cat = create_from_breadcrumbs(cat_id) + cat.code = cat_id + cat.save() + + # Get csv file and open it + csv_file = self.get_csv_fixture("products.csv") + with open(csv_file) as f: + # Use odin codec to load in csv to our created resource + products = csv_codec.reader(f, CSVProductResource, includes_header=True) + + # Map the csv resources to product resources + product_resources = CSVProductMapping.apply(products) + + # Map the product resources to products and save in DB + products_to_db(product_resources) + + self.assertEquals(Product.objects.all().count(), 59) + self.assertEquals(ProductAttributeValue.objects.all().count(), 257) + self.assertEquals(ProductImage.objects.all().count(), 52) + + def get_csv_fixture(self, filename): + return path.realpath( + path.join( + path.dirname(__file__), + "../../", + "oscar_odin/fixtures/oscar_odin/csv/", + filename, + ) + ) From 2c9e2a5d797c6659bd6f302d7d4aa25b8ad3143f Mon Sep 17 00:00:00 2001 From: Viggo de Vries Date: Wed, 20 Dec 2023 12:16:50 +0100 Subject: [PATCH 6/6] Make parent stuff work --- Makefile | 2 +- oscar_odin/exceptions.py | 2 + oscar_odin/mappings/catalogue.py | 39 ++++++------- oscar_odin/mappings/constants.py | 6 +- oscar_odin/mappings/context.py | 20 ++++++- oscar_odin/resources/catalogue.py | 19 +++--- tests/reverse/test_catalogue.py | 94 +++++++++++++++++++++++++++++- tests/reverse/test_reallifecase.py | 5 ++ 8 files changed, 152 insertions(+), 35 deletions(-) create mode 100644 oscar_odin/exceptions.py diff --git a/Makefile b/Makefile index 88d570e..f4ba6dc 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ lint: fail-if-no-virtualenv pylint oscar_odin/ test: fail-if-no-virtualenv - python3 runtests.py test tests.reverse.test_reallifecase + python3 runtests.py test tests.reverse.test_catalogue black: @black oscar_odin/ diff --git a/oscar_odin/exceptions.py b/oscar_odin/exceptions.py new file mode 100644 index 0000000..596c27b --- /dev/null +++ b/oscar_odin/exceptions.py @@ -0,0 +1,2 @@ +class OscarOdinException(TypeError): + pass diff --git a/oscar_odin/mappings/catalogue.py b/oscar_odin/mappings/catalogue.py index b8f6366..1b758ad 100644 --- a/oscar_odin/mappings/catalogue.py +++ b/oscar_odin/mappings/catalogue.py @@ -217,19 +217,6 @@ def attributes(self) -> Dict[str, Any]: for item in self.source.get_attribute_values() } - @odin.assign_field - def children(self) -> Tuple[Optional[List[resources.catalogue.Product]]]: - """Children of parent products.""" - - if self.context.get("include_children", False) and self.source.is_parent: - # Return a tuple as an optional list causes problems. - return ( - map_queryset( - ProductToResource, self.source.children, context=self.context - ), - ) - return (None,) - @odin.assign_field(to_field=("price", "currency", "availability")) def map_stock_price(self) -> Tuple[Decimal, str, int]: """Resolve stock price using strategy and decompose into price/currency/availability.""" @@ -258,15 +245,17 @@ def images(self, values) -> List[ProductImageModel]: """Map related image. We save these later in bulk""" return ProductImageToModel.apply(values) + @odin.map_field + def parent(self, parent): + if parent: + return ParentToModel.apply(parent) + + return None + @odin.map_list_field def categories(self, values) -> List[CategoryModel]: return CategoryToModel.apply(values) - @odin.map_list_field - def children(self, values) -> List[ProductModel]: - """Map related image.""" - return [] - @odin.map_list_field( from_field=["price", "availability", "currency", "upc", "partner"] ) @@ -288,13 +277,19 @@ def stockrecords( @odin.map_field def product_class(self, value) -> ProductClassModel: + if not value and self.source.structure == ProductModel.CHILD: + return None + return ProductClassToModel.apply(value) - # if isinstance(value, self.to_obj): -# return value -# -# return ProductClassToModel.apply(value) +class ParentToModel(odin.Mapping): + from_obj = resources.catalogue.ParentProduct + to_obj = ProductModel + + @odin.assign_field + def structure(self): + return ProductModel.PARENT def product_to_resource_with_strategy( diff --git a/oscar_odin/mappings/constants.py b/oscar_odin/mappings/constants.py index b6fa471..4264716 100644 --- a/oscar_odin/mappings/constants.py +++ b/oscar_odin/mappings/constants.py @@ -2,9 +2,10 @@ Product = get_model("catalogue", "Product") Category = get_model("catalogue", "Category") -StockRecord = get_model("partner", "StockRecord") ProductClass = get_model("catalogue", "ProductClass") ProductImage = get_model("catalogue", "ProductImage") +StockRecord = get_model("partner", "StockRecord") +Partner = get_model("partner", "Partner") PRODUCT_STRUCTURE = "Product.structure" PRODUCT_IS_PUBLIC = "Product.is_public" @@ -16,6 +17,7 @@ PRODUCT_META_TITLE = "Product.meta_title" PRODUCT_META_DESCRIPTION = "Product.meta_description" PRODUCT_PRODUCT_CLASS = "Product.product_class" +PRODUCT_PARENT = "Product.parent" PRODUCT_IS_DISCOUNTABLE = "Product.is_discountable" CATEGORY_NAME = "Category.name" @@ -51,6 +53,7 @@ PRODUCT_META_DESCRIPTION, PRODUCT_PRODUCT_CLASS, PRODUCT_IS_DISCOUNTABLE, + PRODUCT_PARENT, ] ALL_CATEGORY_FIELDS = [ @@ -91,4 +94,5 @@ StockRecord: ("product_id",), ProductClass: ("slug",), ProductImage: ("code",), + Partner: ("slug",), } diff --git a/oscar_odin/mappings/context.py b/oscar_odin/mappings/context.py index 988763d..cc6327d 100644 --- a/oscar_odin/mappings/context.py +++ b/oscar_odin/mappings/context.py @@ -4,6 +4,11 @@ from odin.utils import getmeta from oscar_odin.utils import in_bulk +from oscar_odin.exceptions import OscarOdinException + +from oscar.core.loading import get_model + +Product = get_model("catalogue", "Product") def get_instances_to_create_or_update(Model, instances, identifier_mapping): @@ -71,7 +76,7 @@ def add_instances_to_o2m_relation(self, relation, instances): self.one_to_many_items[relation] += [instances] def add_instance_to_fk_items(self, field, instance): - if not instance.pk: + if instance is not None and not instance.pk: self.foreign_key_items[field] += [instance] def get_fields_to_update(self, Model): @@ -126,8 +131,17 @@ def get_fk_relations(self): relation.related_model, instances, self.identifier_mapping ) - to_create[relation].extend(instances_to_create) - to_update[relation].extend(instances_to_update) + if relation.related_model == Product: + if instances_to_create: + raise OscarOdinException( + "Cannot create parents this way. Please create all parents first seperately, then create the childs while linking the parents using the `oscar_odin.resources.catalogue.ParentProduct`" + ) + + for instance in instances_to_update: + instance.refresh_from_db() + else: + to_create[relation].extend(instances_to_create) + to_update[relation].extend(instances_to_update) return (to_create, to_update) diff --git a/oscar_odin/resources/catalogue.py b/oscar_odin/resources/catalogue.py index e644219..e04284a 100644 --- a/oscar_odin/resources/catalogue.py +++ b/oscar_odin/resources/catalogue.py @@ -93,6 +93,10 @@ class ProductAttributeValue(OscarCatalogue): value: Any +class ParentProduct(OscarCatalogue): + upc: str + + class Product(OscarCatalogue): """A product within Django Oscar.""" @@ -101,12 +105,13 @@ class Product(OscarCatalogue): structure: Structure title: str slug: str - description: str = odin.Options(empty=True) + description: str = "" meta_title: Optional[str] images: List[Image] = odin.Options(empty=True) rating: Optional[float] - is_discountable: bool - is_public: bool + is_discountable: bool = True + is_public: bool = True + parent: Optional[ParentProduct] # Price information price: Decimal = DecimalField() @@ -114,12 +119,12 @@ class Product(OscarCatalogue): availability: Optional[int] partner: Optional[Any] - product_class: ProductClass + product_class: Optional[ProductClass] = None attributes: Dict[str, Any] categories: List[Category] - children: Optional[List["Product"]] = odin.ListOf.delayed( - lambda: Product, null=True - ) + # children: Optional[List["Product"]] = odin.ListOf.delayed( + # lambda: Product, null=True + # ) date_created: datetime date_updated: datetime diff --git a/tests/reverse/test_catalogue.py b/tests/reverse/test_catalogue.py index 261175e..b4dc091 100644 --- a/tests/reverse/test_catalogue.py +++ b/tests/reverse/test_catalogue.py @@ -15,8 +15,9 @@ ProductClass as ProductClassResource, Category as CategoryResource, ProductAttributeValue as ProductAttributeValueResource, + ParentProduct as ParentProductResource, ) - +from oscar_odin.exceptions import OscarOdinException from oscar_odin.mappings.constants import ( STOCKRECORD_PRICE, STOCKRECORD_NUM_IN_STOCK, @@ -404,3 +405,94 @@ def test_create_product_with_related_fields(self): self.assertEqual(prd2.attr.henk, "Klaas") self.assertEqual(prd2.attr.harrie, 1) + + +class ParentChildTest(TestCase): + def test_parent_childs(self): + Category.add_root(name="henk", slug="klaas", is_public=True, code="2") + ProductClass.objects.create( + name="Klaas", slug="klaas", requires_shipping=True, track_stock=True + ) + product_class = ProductClassResource(slug="klaas") + partner = Partner.objects.create(name="klaas") + + prds = ProductResource( + upc="1234323-2", + title="asdf2", + slug="asdf-asdfasdf2", + description="description", + structure=Product.PARENT, + product_class=product_class, + categories=[CategoryResource(code="2")], + ) + + products_to_db(prds) + + prd = Product.objects.get(upc="1234323-2") + + self.assertEquals(prd.structure, Product.PARENT) + self.assertEquals(prd.product_class.slug, "klaas") + + child_product = ProductResource( + parent=ParentProductResource(upc="1234323-2"), + upc="1234323-child", + title="asdf2 child", + slug="asdf-asdfasdf2-child", + structure=Product.CHILD, + price=D("20"), + availability=2, + currency="EUR", + partner=Partner.objects.create(name="klaas"), + ) + + products_to_db(child_product) + + prd = Product.objects.get(upc="1234323-2") + + self.assertEquals(prd.structure, Product.PARENT) + self.assertEquals(prd.product_class.slug, "klaas") + + child = Product.objects.get(upc="1234323-child") + + self.assertEquals(child.structure, Product.CHILD) + self.assertEquals(child.parent.pk, prd.pk) + + def test_non_existing_parent_childs(self): + Category.add_root(name="henk", slug="klaas", is_public=True, code="2") + ProductClass.objects.create( + name="Klaas", slug="klaas", requires_shipping=True, track_stock=True + ) + product_class = ProductClassResource(slug="klaas") + partner = Partner.objects.create(name="klaas") + + prds = ProductResource( + upc="1234323-2", + title="asdf2", + slug="asdf-asdfasdf2", + description="description", + structure=Product.PARENT, + product_class=product_class, + categories=[CategoryResource(code="2")], + ) + + products_to_db(prds) + + prd = Product.objects.get(upc="1234323-2") + + self.assertEquals(prd.structure, Product.PARENT) + self.assertEquals(prd.product_class.slug, "klaas") + + child_product = ProductResource( + parent=ParentProductResource(upc="1234323-654"), + upc="1234323-child", + title="asdf2 child", + slug="asdf-asdfasdf2-child", + structure=Product.CHILD, + price=D("20"), + availability=2, + currency="EUR", + partner=Partner.objects.create(name="klaas"), + ) + + with self.assertRaises(OscarOdinException): + products_to_db(child_product) diff --git a/tests/reverse/test_reallifecase.py b/tests/reverse/test_reallifecase.py index 805d391..4d58183 100644 --- a/tests/reverse/test_reallifecase.py +++ b/tests/reverse/test_reallifecase.py @@ -130,6 +130,11 @@ def is_discountable(self): class RealLifeTest(TestCase): def test_mapping(self): + for partner_id in ["1049", "1052", "1053", "1049"]: + Partner.objects.get_or_create( + code=partner_id, defaults={"name": partner_id} + ) + # Create product class product_class, _ = ProductClass.objects.get_or_create( slug="standard",