Merge pull request 'added CSV upload to enable fast creation of similar components' (#14) from feature/csv-import into develop

Reviewed-on: #14
Reviewed-by: Mario Hüttel <mario.huettel@linux.com>

requirements.txt

@@ -1,5 +1,7 @@
 asgiref==3.4.1
 astroid==2.6.5
+certifi==2024.8.30
+charset-normalizer==3.4.0
 crispy-bootstrap5==0.6
 Django==3.2.5
 django-crispy-forms==1.13.0
@@ -8,19 +10,24 @@ django-qr-code==2.2.0
 django-rest-framework==0.1.0
 django-tex==1.1.9.post1
 djangorestframework==3.12.4
+gunicorn==21.2.0
+idna==3.10
 isort==5.9.3
 Jinja2==3.0.1
 lazy-object-proxy==1.6.0
 MarkupSafe==2.0.1
 mccabe==0.6.1
+packaging==24.2
 Pillow==8.3.1
+psycopg2-binary==2.9.9
 pylint==2.9.6
 pytz==2021.1
 qrcode==7.2
+requests==2.32.3
 segno==1.3.3
+setuptools==75.3.0
 six==1.16.0
 sqlparse==0.4.1
 toml==0.10.2
+urllib3==2.2.3
 wrapt==1.12.1
-psycopg2-binary==2.9.9
-gunicorn==21.2.0

shimatta_kenkyusho/parts/component_import.py

@@ -0,0 +1,110 @@
+import io
+import csv
+import requests
+from django.db import transaction
+from django.core.files.images import ImageFile
+from.models import ComponentParameter, ComponentType, Manufacturer, Component, \
+                    DistributorNum, Stock, Storage, Distributor, Package, ComponentParameterType
+
+def _stock_component(component, storage_uuid, substorage_path, amount, lot, watermark):
+
+    if not amount or not any([storage_uuid, substorage_path]):
+        return None
+
+    storage = Storage.from_path(substorage_path, storage_uuid)
+
+    stock = Stock.objects.create(component=component,
+                                 storage=storage,
+                                 amount=amount,
+                                 lot=lot,
+                                 watermark=watermark)
+    return stock
+
+def _set_additional_parameters(component, type, value):
+
+    if type.startswith('param:'):
+        type = type[6:]
+        param_type = ComponentParameterType.objects.get(parameter_name=type)
+
+        param = ComponentParameter.objects.create(component=component,
+                                                  parameter_type=param_type)
+        if param_type == 'F':
+            param.text_value = value
+        else:
+            param.value = float(value)
+        param.save()
+
+        return param
+
+    elif type.startswith('distri:'):
+        distri_name = type[7:]
+        distri = Distributor.objects.get(name=distri_name)
+
+        distri_num = DistributorNum.objects.create(component=component,
+                                                   distributor=distri,
+                                                   distributor_part_number=value)
+        return distri_num
+
+def import_components_from_csv(csv_file):
+    """
+    Imports components from a csv file containing the component model fields as
+    well as storage information in the heading.
+
+    Parameters can be set by param:<parameter name>, distri numbers by
+    distri:<distri name>.
+    """
+
+    with transaction.atomic():
+
+        # simulate a text-file for the csv lib
+        with io.TextIOWrapper(csv_file, encoding='utf8') as csv_text_file:
+            rows = csv.DictReader(csv_text_file, delimiter=";")
+
+            for data in rows:
+
+                image = None
+                image_url = data.pop('image_url')
+                if image_url:
+                    response = requests.get(image_url)
+                    image_content = response.content
+                    image_file = io.BytesIO(image_content)
+                    image = ImageFile(image_file, 'downloaded_file')
+
+                manufacturer = None
+                manufacturer_name = data.pop('manufacturer')
+                if manufacturer_name:
+                    manufacturer = Manufacturer.objects.get(name=manufacturer_name)
+
+                component_type = None
+                component_type_name = data.pop('component_type')
+                if component_type_name:
+                    component_type = ComponentType.objects.get(class_name=component_type_name)
+
+                distributor = None
+                pref_distri = data.pop('pref_distri')
+                if pref_distri:
+                    distributor = Distributor.objects.get(name=pref_distri)
+
+                package = None
+                package_name = data.pop('package')
+                if package_name:
+                    package = Package.objects.get(name=package_name)
+
+                comp = Component.objects.create(name=data.pop('name'),
+                                                manufacturer=manufacturer,
+                                                component_type=component_type,
+                                                pref_distri=distributor,
+                                                description=data.pop('description'),
+                                                datasheet_link=data.pop('datasheet_link'),
+                                                package=package,
+                                                image=image)
+
+                _stock_component(comp,
+                                 data.pop('storage_uuid'),
+                                 data.pop('substorage_path'),
+                                 data.pop('amount'),
+                                 data.pop('lot'),
+                                 data.pop('watermark'))
+                
+                for key, value in data.items():
+                    _set_additional_parameters(comp, key, value)

shimatta_kenkyusho/parts/forms.py

@@ -214,6 +214,9 @@ class ComponentForm(forms.ModelForm):
         model = parts_models.Component
         fields = '__all__'
 
+class ImportComponentForm(forms.Form):
+    csv_file = forms.FileField(label="CSV File")
+
 class PackageForm(forms.ModelForm):
     class Meta:
         model = parts_models.Package

shimatta_kenkyusho/parts/models.py

@@ -122,6 +122,29 @@ class Storage(models.Model):
 			sum = 0
 		return sum
 
+	@classmethod
+	def from_path(cls, path, root_storage=None):
+		'''
+		Get the storage object described by its complete path or the sub-path
+		from the passed root_storage uuid
+		'''
+		parts = path.split('/')
+
+		# assemble filter query
+		filter_dict = {}
+
+		layer = 0
+		for part in parts[::-1]:
+			filter_dict[f'{"parent_storage__" * layer}name'] = part
+			layer += 1
+
+		if root_storage:
+			filter_dict[f'{"parent_storage__" * layer}id'] = root_storage
+		else:
+			filter_dict[f'{"parent_storage__" * layer}isnull'] = True
+		obj = cls.objects.get(**filter_dict)
+		return obj
+
 	def save(self, *args, **kwargs):
 		self.validate_unique()
 		super(Storage, self).save(*args, **kwargs)

shimatta_kenkyusho/parts/views.py

@@ -7,14 +7,15 @@ from django.contrib.auth.forms import PasswordChangeForm
 from django.contrib.auth import update_session_auth_hash
 import django.forms as forms
 from django.views.generic import TemplateView, DetailView
-from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
-from .models import Storage, Stock, Component, Distributor, Manufacturer, Package, ComponentParameter, ComponentParameterType, DistributorNum
+from django.contrib.auth.mixins import LoginRequiredMixin
+from .models import Storage, Stock, Component, Distributor, Manufacturer, Package, ComponentParameter, DistributorNum
 from .qr_parser import QrCodeValidator
 from django.core.paginator import Paginator
 from django.core.exceptions import ValidationError
 from django.db import IntegrityError
 from django.db.models import ProtectedError
 from .forms import *
+from .component_import import import_components_from_csv
 from django.db.models import Q
 from django.db.models.functions import Lower
 from django.forms import formset_factory
@@ -207,6 +208,7 @@ class ComponentView(LoginRequiredMixin, BaseTemplateMixin, TemplateView):
 
         context['components'] = comp_paginator.get_page(comp_page_num)
         context['comp_form'] = ComponentForm()
+        context['import_comp_form'] = ImportComponentForm()
         context['search_string'] = search
 
         if not parameter_formset:
@@ -221,7 +223,6 @@ class ComponentView(LoginRequiredMixin, BaseTemplateMixin, TemplateView):
     def get_context_data(self, **kwargs):
         return self.get_context_data_int(advanced_search = None, parameter_formset=None, **kwargs)    
 
-
     def handle_new_component_post(self, request, open=False, **kwargs):
         cform = ComponentForm(data=request.POST, files=request.FILES)
         new_component = None
@@ -236,6 +237,20 @@ class ComponentView(LoginRequiredMixin, BaseTemplateMixin, TemplateView):
             return redirect(reverse('parts-components-detail', kwargs={'uuid':new_component.id}))
         return self.render_to_response(context)
 
+    def handle_import_components_post(self, request, open=False, **kwargs):
+        cform = ImportComponentForm(data=request.POST, files=request.FILES)
+        context = self.get_context_data(**kwargs)
+        if cform.is_valid():
+            try:
+                import_components_from_csv(cform.files['csv_file'].file)
+            except Exception as ex:
+                cform.add_error('csv_file', str(ex))
+                context['import_comp_form'] = cform
+        else:
+            context['import_comp_form'] = cform
+
+        return self.render_to_response(context)
+
     def handle_advanced_search_post(self, request, **kwargs):
         
         form = AdvancedComponentSearchForm(auto_id='adv_search_%s', data=request.POST)
@@ -256,6 +271,8 @@ class ComponentView(LoginRequiredMixin, BaseTemplateMixin, TemplateView):
             return self.handle_new_component_post(request, open=False, **kwargs)
         elif 'submit-edit-component-open' in request.POST:
             return self.handle_new_component_post(request, open=True, **kwargs)
+        elif 'submit-import-components' in request.POST:
+            return self.handle_import_components_post(request, open=True, **kwargs)
         elif 'submit-advanced-search' in request.POST:
             return self.handle_advanced_search_post(request, **kwargs)
         else:

shimatta_kenkyusho/static/example/import_csv/import_csv.csv

@@ -0,0 +1,2 @@
+name;manufacturer;component_type;pref_distri;description;datasheet_link;package;image_url;storage_uuid;substorage_path;amount;lot;watermark;param:Resistance;param:Temperature coefficient;param:Tolerance;distri:Mouser
+1K41;Phycomp;Resistor;;KVG - expired;;R0603;https://t3.ftcdn.net/jpg/02/95/44/22/360_F_295442295_OXsXOmLmqBUfZreTnGo9PREuAPSLQhff.jpg;a4909bb4-4d69-4e58-8fbd-fba4fa62d9fb;bew;1111;4000;0;1330;100;1;gggg

shimatta_kenkyusho/static/example/import_csv/readme.md

@@ -0,0 +1,14 @@
+# The shimatta kenkyusho can import components from a CSV file uploaded to the website
+
+All parameters are passed as names (e.g. package, manufacturer etc.).
+
+Component parameters can be set using a prefix ``param:`` followed by the
+parameter name. The value is then parsed bysed on the parameter type.
+
+Distributor part numbers can be set similarly by prepending ``distri:``.
+
+It is also possible to create initial stocks to one storage by passing the
+storage uuid and/or the path to the storage (as printed in the breadcrumbs) or
+a combination of a storage uuid and the path from this storage.
+
+See example for details.

shimatta_kenkyusho/templates/base.html

@@ -78,6 +78,11 @@
         <script type="text/javascript" src="{% static 'js/kenyusho-api-v1.js' %}"></script>
         <script type="text/javascript" src="{% static 'js/autocomplete.js' %}"></script>
         <script type="text/javascript" src="{% static 'js/autocomplete-foreign-key-field.js' %}"></script>
+        <!-- Initialize bootstrap popovers -->
+        <script type="text/javascript">
+            const popoverTriggerList = document.querySelectorAll('[data-bs-toggle="popover"]')
+            const popoverList = [...popoverTriggerList].map(popoverTriggerEl => new bootstrap.Popover(popoverTriggerEl))
+        </script>
         {% block custom_scripts %}
         {% endblock custom_scripts %}
 

shimatta_kenkyusho/templates/parts/components.html

@@ -14,6 +14,7 @@
                     </button>
                     <button class="btn btn-secondary" type="button" data-bs-toggle="collapse" href="#advanced-search-collapse">Advanced <i class="bi bi-search"></i></button>
                     <button class="btn btn-success" type="button" data-bs-toggle="modal" data-bs-target="#comp-edit-modal"><i class="bi bi-plus-circle"></i> Add Component</button>
+                    <button class="btn btn-success" type="button" data-bs-toggle="modal" data-bs-target="#comp-import-modal"><i class="bi bi-plus-circle"></i> Import CSV</button>
                 </div>
             </form>
             <div class="collapse mb-3{% if advanced_search_shown %} show{% endif %}" id="advanced-search-collapse" aria-expanded="{% if advanced_search_shown %}true{% else %}false{% endif %}">
@@ -64,6 +65,7 @@
 </div>
 
 {% include 'parts/modals/edit-component-modal.html' with form=comp_form heading='New Component' open_component_button=True %}
+{% include 'parts/modals/import-component-modal.html' with form=import_comp_form %}
 
 {% endblock content %}
 {% block custom_scripts %}
@@ -72,5 +74,8 @@
 {% if comp_form.errors %}
 bootstrap.Modal.getOrCreateInstance(document.getElementById('comp-edit-modal')).show()
 {% endif %}
+{% if import_comp_form.errors %}
+bootstrap.Modal.getOrCreateInstance(document.getElementById('comp-import-modal')).show()
+{% endif %}
 </script>
 {% endblock custom_scripts %}

shimatta_kenkyusho/templates/parts/modals/import-component-modal.html

@@ -0,0 +1,38 @@
+{% load static %}
+{% load crispy_forms_tags %}
+
+<div class="modal fade" id="comp-import-modal" tabindex="-1">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h2>Import Component CSV</h2>
+                <button type="button"
+                        class="btn"
+                        data-bs-toggle="popover"
+                        data-bs-title="Import Component CSV"
+                        data-bs-html="true"
+                        data-bs-content="The shimatta kenkyusho can import components from a CSV file uploaded to the website.<br>
+                                         All parameters are passed as names (e.g. package, manufacturer etc.).<br><br>
+                                         Component parameters can be set using a prefix <code>param:</code> followed by the
+                                         parameter name. The value is then parsed bysed on the parameter type.<br><br>
+                                         Distributor part numbers can be set similarly by prepending <code>distri:</code>.<br><br>
+                                         It is also possible to create initial stocks to one storage by passing the
+                                         storage uuid and/or the path to the storage (as printed in the breadcrumbs) or
+                                         a combination of a storage uuid and the path from this storage.<br><br>
+                                         See <a href='{% static 'example/import_csv/import_csv.csv' %}'>example</a> for details.">
+                    <i class="bi bi-info-circle"></i>
+                </button>
+                <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+            </div>
+            <form method="post" enctype="multipart/form-data">
+                {% csrf_token %}
+                <div class="modal-body">
+                    {{form|crispy}}
+                </div>
+                <div class="modal-footer">
+                    <input type="submit" name="submit-import-components" class="btn btn-primary" value="Save">
+                </div>
+            </form>
+        </div>
+    </div>
+</div>

shimatta_kenkyusho/templates/parts/stocks-detail.html

@@ -37,7 +37,7 @@
                     <a href="{% url 'parts-stocks-detail' uuid=storage.id %}" class="text-decoration-none">
                         <li class="list-group-item list-group-item-action justify-content-between align-items-center d-flex">
                             <div> 
-                            <h5>{{storage.name}}</h5>
+                            <h5>{{storage.name}}{% if storage.verbose_name %}<small> ({{storage.verbose_name}})</small>{% endif %}</h5>
                             Responsible: {{ storage.responsible }}
                             </div>
                             <span class="badge bg-primary rounded-pill">{{storage.get_total_stock_amount}}</span>