From 635494fc806d70c9184d374e36cc6ad7fc81dc60 Mon Sep 17 00:00:00 2001 From: Jens Martin Norheim Berget Date: Tue, 30 Apr 2024 00:24:41 +0200 Subject: [PATCH] feat: :sparkles: Doubled size of database and computed cosine similarties, now contains 2000 products. --- backend/cal_recommendation.py | 58 ++++++++++++----------------------- backend/db.sqlite3 | 4 +-- backend/transfer_data.py | 6 ++-- 3 files changed, 25 insertions(+), 43 deletions(-) diff --git a/backend/cal_recommendation.py b/backend/cal_recommendation.py index fff678e..4006858 100644 --- a/backend/cal_recommendation.py +++ b/backend/cal_recommendation.py @@ -1,50 +1,35 @@ import os import django -print("Test", flush=True) - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cogitoXInfor.settings') django.setup() -print("Test2", flush=True) - -import csv from GoogleAmazone.models import Recommendations, Products from sentence_transformers import SentenceTransformer, util -Recommendations.objects.all().delete() -print("Test3", flush=True) +import tensorflow as tf +print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) -# Add a recommendation calculation here to precompile recommendations -# and add them to the database in the Recommendations table +# Empty the Recommendations table +Recommendations.objects.all().delete() model = SentenceTransformer("all-MiniLM-L6-v2") -print("Test4", flush=True) - all_products = Products.objects.all() -print("All products = ", all_products, flush=True) -product_titles = [product.title for product in all_products] -product_descriptions = [product.description for product in all_products] -# Matching product id's with product titles +# Matching product id's with product titles and descriptions id_title_embed = {} id_desc_embed = {} i = 0 -print("All products length = ", len(all_products), flush=True) for product in all_products: i += 1 - print(i, flush=True) + print("Embedding product number: " + i, flush=True) if product.title == "" or product.description == "": continue id_title_embed[product.id] = model.encode(product.title) id_desc_embed[product.id] = model.encode(product.description) -print("Begynner å embedde", flush=True) -# print("id_title_embed.items() = ", id_title_embed.items(), flush=True) -# print("id_desc_embed.items() = ", id_desc_embed.items(), flush=True) - -# Compute the cosine similarity between the title and description of all pairs of products and store the results in dictionaries +# Compute the cosine similarity between the title and description of all pairs of products and store the results in two distinct dictionaries i = 0 cos_sim_title = {} for id1, title1 in list(id_title_embed.items()): @@ -52,8 +37,11 @@ i += 1 print("Iterasjon (title): " + str(i), flush=True) if id1 != id2: - if (id1, id2) not in cos_sim_title and (id2, id1) not in cos_sim_title: - cos_sim_title[(id1, id2)] = util.cos_sim(title1, title2) + try: + if (id1, id2) not in cos_sim_title and (id2, id1) not in cos_sim_title: + cos_sim_title[(id1, id2)] = util.cos_sim(title1, title2) + except: + print("Error with product-pair: " + id1 + "," + id2, flush=True) i = 0 cos_sim_desc = {} @@ -62,11 +50,15 @@ i += 1 print("Iterasjon (description): " + str(i), flush=True) if id1 != id2: - if (id1, id2) not in cos_sim_desc and (id2, id1) not in cos_sim_desc: - cos_sim_desc[(id1, id2)] = util.cos_sim(desc1, desc2) + try: + if (id1, id2) not in cos_sim_desc and (id2, id1) not in cos_sim_desc: + cos_sim_desc[(id1, id2)] = util.cos_sim(desc1, desc2) + except: + print("Error with product-pair: " + id1 + "," + id2, flush=True) combined_dict = {} +# Create a dictionary with the combined cosine similarities of the title and description embeddings i = 0 for key, value in cos_sim_title.items(): i += 1 @@ -74,6 +66,7 @@ if key in cos_sim_desc: combined_dict[key] = [value, cos_sim_desc[key]] # Struktur: (embedding title, embedding description) +# Save the recommendations to the database i = 0 for pair, sim_title in list(combined_dict.items()): i += 1 @@ -83,15 +76,4 @@ recommendation = Recommendations(col=str(pair[0]), row=str(pair[1]), title_similarity=float(sim_combined[0]), description_similarity=float(sim_combined[1])) recommendation.save() -print("Recommendations saved to the database", flush=True) -# print("Cosine similarities titles = ", cos_sim_title, flush=True) -# print("", flush=True) -# print("Cosine similarities descriptions = ", cos_sim_desc, flush=True) -# print("", flush=True) - - -# print("Combined dictionary = ", combined_dict, flush=True) - - - - +print("Recommendations saved to the database", flush=True) \ No newline at end of file diff --git a/backend/db.sqlite3 b/backend/db.sqlite3 index 8e6d2c3..cf56b12 100644 --- a/backend/db.sqlite3 +++ b/backend/db.sqlite3 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8b0085f81f5c6771160b1196f6b05119a51bccb1ea3ec159c2a250b06c338683 -size 164569088 +oid sha256:f660cd888ec99ab835a6afb30578e2b919f28c570cf192ec1e7cc7d77d0046f6 +size 644423680 diff --git a/backend/transfer_data.py b/backend/transfer_data.py index 8c61dd7..0d63f91 100644 --- a/backend/transfer_data.py +++ b/backend/transfer_data.py @@ -14,13 +14,13 @@ def import_products_from_csv(file_path): with open(file_path, 'r', encoding='utf-8') as csv_file: csv_reader = csv.DictReader(csv_file) - for row in islice(csv_reader, 1000): + for row in islice(csv_reader, 2000): price = row['price'].replace('£', '').strip() try: price = float(price) except ValueError: - price = -1 + price = -1 product = Products.objects.create( id=row['uniq_id'], @@ -31,7 +31,7 @@ def import_products_from_csv(file_path): ) product.save() print(f"Product {product.title} has been created") - print("1000 products have been imported") + print("2000 products have been imported") csv_file_path = "./../data/better_dataset_toys.csv" import_products_from_csv(csv_file_path)