Skip to content

feat: ✨ Doubled size of database #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jmnorheim wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: ✨ Doubled size of database #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 20 additions & 38 deletions backend/cal_recommendation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,59 +1,47 @@
import os
import django

print("Test", flush=True)

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cogitoXInfor.settings')
django.setup()

print("Test2", flush=True)

import csv
from GoogleAmazone.models import Recommendations, Products
from sentence_transformers import SentenceTransformer, util

Recommendations.objects.all().delete()
print("Test3", flush=True)
import tensorflow as tf
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))

# Add a recommendation calculation here to precompile recommendations
# and add them to the database in the Recommendations table
# Empty the Recommendations table
Recommendations.objects.all().delete()

model = SentenceTransformer("all-MiniLM-L6-v2")

print("Test4", flush=True)

all_products = Products.objects.all()
print("All products = ", all_products, flush=True)
product_titles = [product.title for product in all_products]
product_descriptions = [product.description for product in all_products]

# Matching product id's with product titles
# Matching product id's with product titles and descriptions
id_title_embed = {}
id_desc_embed = {}
i = 0
print("All products length = ", len(all_products), flush=True)
for product in all_products:
i += 1
print(i, flush=True)
print("Embedding product number: " + i, flush=True)
if product.title == "" or product.description == "":
continue
id_title_embed[product.id] = model.encode(product.title)
id_desc_embed[product.id] = model.encode(product.description)

print("Begynner å embedde", flush=True)
# print("id_title_embed.items() = ", id_title_embed.items(), flush=True)
# print("id_desc_embed.items() = ", id_desc_embed.items(), flush=True)

# Compute the cosine similarity between the title and description of all pairs of products and store the results in dictionaries
# Compute the cosine similarity between the title and description of all pairs of products and store the results in two distinct dictionaries
i = 0
cos_sim_title = {}
for id1, title1 in list(id_title_embed.items()):
for id2, title2 in list(id_title_embed.items()):
i += 1
print("Iterasjon (title): " + str(i), flush=True)
if id1 != id2:
if (id1, id2) not in cos_sim_title and (id2, id1) not in cos_sim_title:
cos_sim_title[(id1, id2)] = util.cos_sim(title1, title2)
try:
if (id1, id2) not in cos_sim_title and (id2, id1) not in cos_sim_title:
cos_sim_title[(id1, id2)] = util.cos_sim(title1, title2)
except:
print("Error with product-pair: " + id1 + "," + id2, flush=True)

i = 0
cos_sim_desc = {}
Expand All @@ -62,18 +50,23 @@
i += 1
print("Iterasjon (description): " + str(i), flush=True)
if id1 != id2:
if (id1, id2) not in cos_sim_desc and (id2, id1) not in cos_sim_desc:
cos_sim_desc[(id1, id2)] = util.cos_sim(desc1, desc2)
try:
if (id1, id2) not in cos_sim_desc and (id2, id1) not in cos_sim_desc:
cos_sim_desc[(id1, id2)] = util.cos_sim(desc1, desc2)
except:
print("Error with product-pair: " + id1 + "," + id2, flush=True)

combined_dict = {}

# Create a dictionary with the combined cosine similarities of the title and description embeddings
i = 0
for key, value in cos_sim_title.items():
i += 1
print("Iterasjon (combined): " + str(i), flush=True)
if key in cos_sim_desc:
combined_dict[key] = [value, cos_sim_desc[key]] # Struktur: (embedding title, embedding description)

# Save the recommendations to the database
i = 0
for pair, sim_title in list(combined_dict.items()):
i += 1
Expand All @@ -83,15 +76,4 @@
recommendation = Recommendations(col=str(pair[0]), row=str(pair[1]), title_similarity=float(sim_combined[0]), description_similarity=float(sim_combined[1]))
recommendation.save()

print("Recommendations saved to the database", flush=True)
# print("Cosine similarities titles = ", cos_sim_title, flush=True)
# print("", flush=True)
# print("Cosine similarities descriptions = ", cos_sim_desc, flush=True)
# print("", flush=True)


# print("Combined dictionary = ", combined_dict, flush=True)




print("Recommendations saved to the database", flush=True)
4 changes: 2 additions & 2 deletions backend/db.sqlite3
View file
Open in desktop
Git LFS file not shown
6 changes: 3 additions & 3 deletions backend/transfer_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@ def import_products_from_csv(file_path):

with open(file_path, 'r', encoding='utf-8') as csv_file:
csv_reader = csv.DictReader(csv_file)
for row in islice(csv_reader, 1000):
for row in islice(csv_reader, 2000):

price = row['price'].replace('£', '').strip()
try:
price = float(price)
except ValueError:
price = -1
price = -1

product = Products.objects.create(
id=row['uniq_id'],
Expand All @@ -31,7 +31,7 @@ def import_products_from_csv(file_path):
)
product.save()
print(f"Product {product.title} has been created")
print("1000 products have been imported")
print("2000 products have been imported")

csv_file_path = "./../data/better_dataset_toys.csv"
import_products_from_csv(csv_file_path)