From 20a83c7e91989eee68983a40b1c91d0b091cfdaf Mon Sep 17 00:00:00 2001 From: stefan Date: Sun, 17 Nov 2024 19:54:15 +0100 Subject: [PATCH 1/2] added CSV upload to enable fast creation of similar components --- examples/import_csv/add_comps.csv | 2 + examples/import_csv/readme.md | 14 +++ requirements.txt | 11 +- shimatta_kenkyusho/parts/component_import.py | 110 ++++++++++++++++++ shimatta_kenkyusho/parts/forms.py | 5 +- shimatta_kenkyusho/parts/models.py | 23 ++++ shimatta_kenkyusho/parts/views.py | 25 +++- .../templates/parts/components.html | 5 + .../parts/modals/import-component-modal.html | 22 ++++ .../templates/parts/stocks-detail.html | 2 +- 10 files changed, 211 insertions(+), 8 deletions(-) create mode 100644 examples/import_csv/add_comps.csv create mode 100644 examples/import_csv/readme.md create mode 100644 shimatta_kenkyusho/parts/component_import.py create mode 100644 shimatta_kenkyusho/templates/parts/modals/import-component-modal.html diff --git a/examples/import_csv/add_comps.csv b/examples/import_csv/add_comps.csv new file mode 100644 index 0000000..761f02f --- /dev/null +++ b/examples/import_csv/add_comps.csv @@ -0,0 +1,2 @@ +name;manufacturer;component_type;pref_distri;description;datasheet_link;package;image_url;storage_uuid;substorage_path;amount;lot;watermark;param:Resistance;param:Temperature coefficient;param:Tolerance;distri:Mouser +1K41;Phycomp;Resistor;;KVG - expired;;R0603;https://t3.ftcdn.net/jpg/02/95/44/22/360_F_295442295_OXsXOmLmqBUfZreTnGo9PREuAPSLQhff.jpg;a4909bb4-4d69-4e58-8fbd-fba4fa62d9fb;bew;1111;4000;0;1330;100;1;gggg diff --git a/examples/import_csv/readme.md b/examples/import_csv/readme.md new file mode 100644 index 0000000..a763a6c --- /dev/null +++ b/examples/import_csv/readme.md @@ -0,0 +1,14 @@ +# The shimatta kenkyusho can import components from a CSV file uploaded to the website + +All parameters are passed as names (e.g. package, manufacturer etc.). + +Component parameters can be set using a prefix ``param:`` followed by the +parameter name. The value is then parsed bysed on the parameter type. + +Distributor part numbers can be set similarly by prepending ``distri:``. + +It is also possible to create initial stocks to one storage by passing the +storage uuid and/or the path to the storage (as printed in the breadcrumbs) or +a combination of a storage uuid and the path from this storage. + +See example for details. diff --git a/requirements.txt b/requirements.txt index 4a6a537..ba086c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ asgiref==3.4.1 astroid==2.6.5 +certifi==2024.8.30 +charset-normalizer==3.4.0 crispy-bootstrap5==0.6 Django==3.2.5 django-crispy-forms==1.13.0 @@ -8,19 +10,24 @@ django-qr-code==2.2.0 django-rest-framework==0.1.0 django-tex==1.1.9.post1 djangorestframework==3.12.4 +gunicorn==21.2.0 +idna==3.10 isort==5.9.3 Jinja2==3.0.1 lazy-object-proxy==1.6.0 MarkupSafe==2.0.1 mccabe==0.6.1 +packaging==24.2 Pillow==8.3.1 +psycopg2-binary==2.9.9 pylint==2.9.6 pytz==2021.1 qrcode==7.2 +requests==2.32.3 segno==1.3.3 +setuptools==75.3.0 six==1.16.0 sqlparse==0.4.1 toml==0.10.2 +urllib3==2.2.3 wrapt==1.12.1 -psycopg2-binary==2.9.9 -gunicorn==21.2.0 diff --git a/shimatta_kenkyusho/parts/component_import.py b/shimatta_kenkyusho/parts/component_import.py new file mode 100644 index 0000000..902b1d9 --- /dev/null +++ b/shimatta_kenkyusho/parts/component_import.py @@ -0,0 +1,110 @@ +import io +import csv +import requests +from django.db import transaction +from django.core.files.images import ImageFile +from.models import ComponentParameter, ComponentType, Manufacturer, Component, \ + DistributorNum, Stock, Storage, Distributor, Package, ComponentParameterType + +def _stock_component(component, storage_uuid, substorage_path, amount, lot, watermark): + + if not amount or not any([storage_uuid, substorage_path]): + return None + + storage = Storage.from_path(substorage_path, storage_uuid) + + stock = Stock.objects.create(component=component, + storage=storage, + amount=amount, + lot=lot, + watermark=watermark) + return stock + +def _set_additional_parameters(component, type, value): + + if type.startswith('param:'): + type = type[6:] + param_type = ComponentParameterType.objects.get(parameter_name=type) + + param = ComponentParameter.objects.create(component=component, + parameter_type=param_type) + if param_type == 'F': + param.text_value = value + else: + param.value = float(value) + param.save() + + return param + + elif type.startswith('distri:'): + distri_name = type[7:] + distri = Distributor.objects.get(name=distri_name) + + distri_num = DistributorNum.objects.create(component=component, + distributor=distri, + distributor_part_number=value) + return distri_num + +def import_components_from_csv(csv_file): + """ + Imports components from a csv file containing the component model fields as + well as storage information in the heading. + + Parameters can be set by param:, distri numbers by + distri:. + """ + + with transaction.atomic(): + + # simulate a text-file for the csv lib + with io.TextIOWrapper(csv_file, encoding='utf8') as csv_text_file: + rows = csv.DictReader(csv_text_file, delimiter=";") + + for data in rows: + + image = None + image_url = data.pop('image_url') + if image_url: + response = requests.get(image_url) + image_content = response.content + image_file = io.BytesIO(image_content) + image = ImageFile(image_file, 'downloaded_file') + + manufacturer = None + manufacturer_name = data.pop('manufacturer') + if manufacturer_name: + manufacturer = Manufacturer.objects.get(name=manufacturer_name) + + component_type = None + component_type_name = data.pop('component_type') + if component_type_name: + component_type = ComponentType.objects.get(class_name=component_type_name) + + distributor = None + pref_distri = data.pop('pref_distri') + if pref_distri: + distributor = Distributor.objects.get(name=pref_distri) + + package = None + package_name = data.pop('package') + if package_name: + package = Package.objects.get(name=package_name) + + comp = Component.objects.create(name=data.pop('name'), + manufacturer=manufacturer, + component_type=component_type, + pref_distri=distributor, + description=data.pop('description'), + datasheet_link=data.pop('datasheet_link'), + package=package, + image=image) + + _stock_component(comp, + data.pop('storage_uuid'), + data.pop('substorage_path'), + data.pop('amount'), + data.pop('lot'), + data.pop('watermark')) + + for key, value in data.items(): + _set_additional_parameters(comp, key, value) diff --git a/shimatta_kenkyusho/parts/forms.py b/shimatta_kenkyusho/parts/forms.py index d1edc64..7c7b392 100644 --- a/shimatta_kenkyusho/parts/forms.py +++ b/shimatta_kenkyusho/parts/forms.py @@ -213,7 +213,10 @@ class ComponentForm(forms.ModelForm): class Meta: model = parts_models.Component fields = '__all__' - + +class ImportComponentForm(forms.Form): + csv_file = forms.FileField(label="CSV File") + class PackageForm(forms.ModelForm): class Meta: model = parts_models.Package diff --git a/shimatta_kenkyusho/parts/models.py b/shimatta_kenkyusho/parts/models.py index b78ebf8..ee10485 100644 --- a/shimatta_kenkyusho/parts/models.py +++ b/shimatta_kenkyusho/parts/models.py @@ -122,6 +122,29 @@ class Storage(models.Model): sum = 0 return sum + @classmethod + def from_path(cls, path, root_storage=None): + ''' + Get the storage object described by its complete path or the sub-path + from the passed root_storage uuid + ''' + parts = path.split('/') + + # assemble filter query + filter_dict = {} + + layer = 0 + for part in parts[::-1]: + filter_dict[f'{"parent_storage__" * layer}name'] = part + layer += 1 + + if root_storage: + filter_dict[f'{"parent_storage__" * layer}id'] = root_storage + else: + filter_dict[f'{"parent_storage__" * layer}isnull'] = True + obj = cls.objects.get(**filter_dict) + return obj + def save(self, *args, **kwargs): self.validate_unique() super(Storage, self).save(*args, **kwargs) diff --git a/shimatta_kenkyusho/parts/views.py b/shimatta_kenkyusho/parts/views.py index 6de8672..7cf3562 100644 --- a/shimatta_kenkyusho/parts/views.py +++ b/shimatta_kenkyusho/parts/views.py @@ -7,14 +7,15 @@ from django.contrib.auth.forms import PasswordChangeForm from django.contrib.auth import update_session_auth_hash import django.forms as forms from django.views.generic import TemplateView, DetailView -from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin -from .models import Storage, Stock, Component, Distributor, Manufacturer, Package, ComponentParameter, ComponentParameterType, DistributorNum +from django.contrib.auth.mixins import LoginRequiredMixin +from .models import Storage, Stock, Component, Distributor, Manufacturer, Package, ComponentParameter, DistributorNum from .qr_parser import QrCodeValidator from django.core.paginator import Paginator from django.core.exceptions import ValidationError from django.db import IntegrityError from django.db.models import ProtectedError from .forms import * +from .component_import import import_components_from_csv from django.db.models import Q from django.db.models.functions import Lower from django.forms import formset_factory @@ -205,6 +206,7 @@ class ComponentView(LoginRequiredMixin, BaseTemplateMixin, TemplateView): context['components'] = comp_paginator.get_page(comp_page_num) context['comp_form'] = ComponentForm() + context['import_comp_form'] = ImportComponentForm() context['search_string'] = search if not parameter_formset: @@ -218,7 +220,6 @@ class ComponentView(LoginRequiredMixin, BaseTemplateMixin, TemplateView): def get_context_data(self, **kwargs): return self.get_context_data_int(advanced_search = None, parameter_formset=None, **kwargs) - def handle_new_component_post(self, request, open=False, **kwargs): cform = ComponentForm(data=request.POST, files=request.FILES) @@ -233,7 +234,21 @@ class ComponentView(LoginRequiredMixin, BaseTemplateMixin, TemplateView): if open and new_component: return redirect(reverse('parts-components-detail', kwargs={'uuid':new_component.id})) return self.render_to_response(context) - + + def handle_import_components_post(self, request, open=False, **kwargs): + cform = ImportComponentForm(data=request.POST, files=request.FILES) + context = self.get_context_data(**kwargs) + if cform.is_valid(): + try: + import_components_from_csv(cform.files['csv_file'].file) + except Exception as ex: + cform.add_error('csv_file', str(ex)) + context['import_comp_form'] = cform + else: + context['import_comp_form'] = cform + + return self.render_to_response(context) + def handle_advanced_search_post(self, request, **kwargs): form = AdvancedComponentSearchForm(auto_id='adv_search_%s', data=request.POST) @@ -254,6 +269,8 @@ class ComponentView(LoginRequiredMixin, BaseTemplateMixin, TemplateView): return self.handle_new_component_post(request, open=False, **kwargs) elif 'submit-edit-component-open' in request.POST: return self.handle_new_component_post(request, open=True, **kwargs) + elif 'submit-import-components' in request.POST: + return self.handle_import_components_post(request, open=True, **kwargs) elif 'submit-advanced-search' in request.POST: return self.handle_advanced_search_post(request, **kwargs) else: diff --git a/shimatta_kenkyusho/templates/parts/components.html b/shimatta_kenkyusho/templates/parts/components.html index c031198..5c7b97b 100644 --- a/shimatta_kenkyusho/templates/parts/components.html +++ b/shimatta_kenkyusho/templates/parts/components.html @@ -14,6 +14,7 @@ +
@@ -64,6 +65,7 @@
{% include 'parts/modals/edit-component-modal.html' with form=comp_form heading='New Component' open_component_button=True %} +{% include 'parts/modals/import-component-modal.html' with form=import_comp_form %} {% endblock content %} {% block custom_scripts %} @@ -72,5 +74,8 @@ {% if comp_form.errors %} bootstrap.Modal.getOrCreateInstance(document.getElementById('comp-edit-modal')).show() {% endif %} +{% if import_comp_form.errors %} +bootstrap.Modal.getOrCreateInstance(document.getElementById('comp-import-modal')).show() +{% endif %} {% endblock custom_scripts %} \ No newline at end of file diff --git a/shimatta_kenkyusho/templates/parts/modals/import-component-modal.html b/shimatta_kenkyusho/templates/parts/modals/import-component-modal.html new file mode 100644 index 0000000..9ab72fb --- /dev/null +++ b/shimatta_kenkyusho/templates/parts/modals/import-component-modal.html @@ -0,0 +1,22 @@ +{% load static %} +{% load crispy_forms_tags %} + + diff --git a/shimatta_kenkyusho/templates/parts/stocks-detail.html b/shimatta_kenkyusho/templates/parts/stocks-detail.html index 4d969f6..99ee247 100644 --- a/shimatta_kenkyusho/templates/parts/stocks-detail.html +++ b/shimatta_kenkyusho/templates/parts/stocks-detail.html @@ -37,7 +37,7 @@
  • -
    {{storage.name}}
    +
    {{storage.name}}{% if storage.verbose_name %} ({{storage.verbose_name}}){% endif %}
    Responsible: {{ storage.responsible }}
    {{storage.get_total_stock_amount}} From 3d63357534e02da201158ab403ee22cbcf20ba7e Mon Sep 17 00:00:00 2001 From: stefan Date: Sun, 17 Nov 2024 21:44:30 +0100 Subject: [PATCH 2/2] made the readme and example for the CSV import accessible from the form directly --- .../static/example/import_csv/import_csv.csv | 0 .../static/example}/import_csv/readme.md | 0 shimatta_kenkyusho/templates/base.html | 5 +++++ .../parts/modals/import-component-modal.html | 16 ++++++++++++++++ 4 files changed, 21 insertions(+) rename examples/import_csv/add_comps.csv => shimatta_kenkyusho/static/example/import_csv/import_csv.csv (100%) rename {examples => shimatta_kenkyusho/static/example}/import_csv/readme.md (100%) diff --git a/examples/import_csv/add_comps.csv b/shimatta_kenkyusho/static/example/import_csv/import_csv.csv similarity index 100% rename from examples/import_csv/add_comps.csv rename to shimatta_kenkyusho/static/example/import_csv/import_csv.csv diff --git a/examples/import_csv/readme.md b/shimatta_kenkyusho/static/example/import_csv/readme.md similarity index 100% rename from examples/import_csv/readme.md rename to shimatta_kenkyusho/static/example/import_csv/readme.md diff --git a/shimatta_kenkyusho/templates/base.html b/shimatta_kenkyusho/templates/base.html index 298c945..f7daf6b 100644 --- a/shimatta_kenkyusho/templates/base.html +++ b/shimatta_kenkyusho/templates/base.html @@ -78,6 +78,11 @@ + + {% block custom_scripts %} {% endblock custom_scripts %} diff --git a/shimatta_kenkyusho/templates/parts/modals/import-component-modal.html b/shimatta_kenkyusho/templates/parts/modals/import-component-modal.html index 9ab72fb..62acb6d 100644 --- a/shimatta_kenkyusho/templates/parts/modals/import-component-modal.html +++ b/shimatta_kenkyusho/templates/parts/modals/import-component-modal.html @@ -6,6 +6,22 @@

    Import Component CSV

    +