diff --git a/app/configuration.py b/app/configuration.py index 092b4a8..150124b 100644 --- a/app/configuration.py +++ b/app/configuration.py @@ -14,5 +14,6 @@ class Config(object): class DevelopmentConfig(Config): DEBUG = True + class TestingConfig(Config): TESTING = True diff --git a/app/forms.py b/app/forms.py index 5cf3bcd..b85c80e 100644 --- a/app/forms.py +++ b/app/forms.py @@ -3,19 +3,19 @@ from flask_wtf import FlaskForm from wtforms import ( StringField, SubmitField, - IntegerField, HiddenField, FloatField, URLField, + TextAreaField, ) from wtforms.validators import DataRequired class NewWishlist(FlaskForm): - title = StringField("Title:", validators=[DataRequired()]) - description = StringField("Description:", validators=[DataRequired()]) - submit = SubmitField("Submit") + title = StringField("Title", validators=[DataRequired()]) + description = TextAreaField("Description", validators=[DataRequired()]) + submit = SubmitField("Create") # Each submit needs a different page fot it to work on the same page. @@ -24,8 +24,8 @@ class DeleteWishlist(FlaskForm): class EditWishlistInfo(FlaskForm): - title = StringField("Title", validators=[DataRequired()]) - description = StringField("Description", validators=[DataRequired()]) + title = StringField("Title") + description = TextAreaField("Description") wl_edit_submit = SubmitField("Submit") @@ -34,7 +34,7 @@ class ResetWishlistUrls(FlaskForm): class NewItem(FlaskForm): - title = StringField("Title", validators=[DataRequired()]) + it_new_title = StringField("Title", validators=[DataRequired()]) description = StringField("Description", validators=[DataRequired()]) price = FloatField("Price", validators=[DataRequired()]) url = URLField("Url", validators=[DataRequired()]) @@ -48,7 +48,6 @@ class CheckItem(FlaskForm): class DeleteItem(FlaskForm): index = HiddenField() - it_del_submit = SubmitField("Delete item") def parseHiddenIndex(field: HiddenField, array: list[Any]) -> int | None: diff --git a/app/scrapers.py b/app/scrapers.py new file mode 100644 index 0000000..34bda9d --- /dev/null +++ b/app/scrapers.py @@ -0,0 +1,166 @@ +from abc import ABC, abstractmethod +from typing import Callable, override +from bs4 import BeautifulSoup +from requests import get +from re import findall, match, search + +noReturnLambda: Callable[[str], str] = lambda x: x + + +class ScrapeError(Exception): + pass + + +class ScraperResult: + def __init__(self, name: str, price: float, image: str): + self.name = name + self.price = price + self.image = image + + @override + def __repr__(self) -> str: + return ( + f"" + ) + + name: str + price: float + image: str + + +class ScraperLike(ABC): + name: str + urlRegex: str + + @abstractmethod + def scrape(self, url: str) -> ScraperResult: + pass + + +class GenericScraper(ScraperLike): + name: str + urlRegex: str + _nameQuery: str + _priceQuery: str + _imageQuery: str + priceParser: Callable[[str], str] + imageParser: Callable[[str], str] + + def __init__( + self, + name: str, + baseUrl: str, + nameQuery: str, + priceQuery: str, + imageQuery: str, + priceParser: Callable[[str], str] = noReturnLambda, + imageParser: Callable[[str], str] = noReturnLambda, + ): + self.name = name + self.urlRegex = baseUrl + self._nameQuery = nameQuery + self._priceQuery = priceQuery + self._imageQuery = imageQuery + self.priceParser = priceParser + self.imageParser = imageParser + + @override + def scrape(self, url: str) -> ScraperResult: + res = get( + url, + headers={ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" + }, + ) + if res.status_code != 200: + raise ScrapeError("Failed to fetch page.") + + soup = BeautifulSoup(res.text, features="html.parser") + name = soup.select_one(self._nameQuery) + price = soup.select_one(self._priceQuery) + image = soup.select_one(self._imageQuery) + + if name is None or image is None: + raise ScrapeError( + f"Failed to scrape site. Invalid webpage or queries: N:{name},P:{price},I:{image}" + ) + + name = name.text.strip() + image = image.get("src") + try: + if price is None: + price = "0" + else: + price = price.text + x = self.priceParser(price) + reg = search(r"([0-9]+)(?:(?:\.|,)([0-9]+))?", x) + if not reg: + raise ValueError + x = reg.group(1) + + g2 = reg.group(2) + if g2: + x += "." + g2 + + price = float(x) + except ValueError: + print(price) + raise ScrapeError(f"Failed to scrape site. Error while parsing price.") + if not isinstance(image, str): + raise ScrapeError(f"Failed to scrape site. Error while parsing image.") + + return ScraperResult(name, price, self.imageParser(image)) + + +def scrapeSite(url: str) -> ScraperResult | None: + scraped: ScraperResult | None = None + + for i in scrapers: + if match(i.urlRegex, url) is None: + continue + + scraped = i.scrape(url) + + return scraped + + +scrapers = [ + GenericScraper( + "Amazon", + r"^https?:\/\/(www\.)?((amazon)|(amzn))\.\w*", + "#productTitle", + "#corePrice_feature_div > div:nth-child(1) > div:nth-child(1) > span:nth-child(1) > span:nth-child(1)", + "#landingImage", + ), + GenericScraper( + "Bol.com", + r"^https?:\/\/(www\.)?bol.com", + ".page-heading > span:nth-child(1)", + ".promo-price", + "div.container-item:nth-child(2) > wsp-selected-item-image-zoom-modal-application:nth-child(1) > button:nth-child(2) > img:nth-child(1)", + priceParser=lambda x: x.replace("\n ", "."), + ), + GenericScraper( + "MediaMarkt", + r"^https?:\/\/(www\.)?mediamarkt.\w*", + "h1.sc-d571b66f-0", + ".sc-6db49389-0 > span:nth-child(2)", + "div.sc-hLBbgP:nth-child(2) > div:nth-child(3) > ul:nth-child(1) > li:nth-child(1) > div:nth-child(1) > div:nth-child(1) > button:nth-child(1) > img:nth-child(1)", + priceParser=lambda x: x.replace("€", ""), + ), + GenericScraper( + "Coolblue", + r"^https?:\/\/(www\.)?coolblue.\w*", + ".css-1o2kclk", + ".css-puih25 > span:nth-child(1)", + ".css-ptvba5", + ), + GenericScraper( + "Megekko", + r"^https?:\/\/(www\.)?megekko.nl", + "#prd_title", + "a.prsPrice:nth-child(1) > div:nth-child(1)", + "#prd_afbeeldingen > div:nth-child(1) > img:nth-child(1)", + imageParser=lambda x: f"https://www.megekko.nl/{x}", + ), +] diff --git a/app/templates/base.html b/app/templates/base.html new file mode 100644 index 0000000..f3ac209 --- /dev/null +++ b/app/templates/base.html @@ -0,0 +1,67 @@ + + + + {% block head %} + + + Wishthat + + + {% endblock head %} + + + + {% block aboveNavbar %} + {% endblock aboveNavbar %} + +
+ + + {% block content %} + {% endblock content %} +
+ + diff --git a/app/templates/components.html b/app/templates/components.html new file mode 100644 index 0000000..c3a910c --- /dev/null +++ b/app/templates/components.html @@ -0,0 +1,5 @@ +{% macro mainCenter() %} +
+ {{ caller() }} +
+{% endmacro %} diff --git a/app/templates/edit.html b/app/templates/edit.html index 263a54e..1d877de 100644 --- a/app/templates/edit.html +++ b/app/templates/edit.html @@ -1,98 +1,263 @@ {% set cpath = url_for("edit", id=wishlist.editId) %} - -

Metadata

-
- {{ form_wl_editinfo.hidden_tag() }} - - {{ form_wl_editinfo.title.label }} - {{ form_wl_editinfo.title(placeholder=wishlist.title) }} - - - {{ form_wl_editinfo.description.label }} - {{ form_wl_editinfo.description(placeholder=wishlist.description) }} - - - {{ form_wl_editinfo.wl_edit_submit() }} -
- -
-

Urls

- - - -
- {{ form_wl_reseturls.hidden_tag() }} - {{ form_wl_reseturls.wl_reset_submit() }} -
- -
-

New item

-
- {{ form_it_new.hidden_tag() }} - - - {{ form_it_new.title.label }} - {{ form_it_new.title() }} - - - {{ form_it_new.description.label }} - {{ form_it_new.description() }} - - - {{ form_it_new.price.label }} - {{ form_it_new.price() }} - - - {{ form_it_new.url.label }} - {{ form_it_new.url() }} - - - {{ form_it_new.image.label }} - {{ form_it_new.image() }} - - - {{ form_it_new.it_new_submit() }} -
- -
-

Delete items

- -{% if wishlist.items|length == 0 %} -

No items yet

-{% endif %} - +
+
+
+ +

Info

+
+ + {{ form_wl_editinfo.title.label.text }} + {{ form_wl_editinfo.title(placeholder=wishlist.title, class="w-full + input validator mt-1 mb-4") }} + + {{ form_wl_editinfo.description.label.text }} + {{ form_wl_editinfo.description(placeholder=wishlist.description, + class="w-full textarea validator mt-1 mb-2") }} +
Please make sure that both inputs are filled.
+ +
+ + {{ form_wl_delete.hidden_tag() }} {{ form_wl_delete.wl_del_submit(class="btn btn-soft btn-error + join-item w-full") }} + + {{ form_wl_editinfo.wl_edit_submit(class="btn btn-soft btn-success + join-item w-full") }} +
+ +
+
+ +

Urls

+
    +
  • + + + + + + + + + + + +

    {{ wishlist.viewId }}

    + + + + + +
    • +
  • + + + +

    {{ wishlist.editId }}

    + + + + + +
    • +
+
+ {{ form_wl_reseturls.hidden_tag() }} {{ form_wl_reseturls.wl_reset_submit(class="btn btn-soft btn-warning") + }} +
+
+
+ +

Add/Delete items

+
    +
  • + Create item + + + +
    • + {% for value in wishlist.items %} +
  • +
    + +
    +
    {{ value.title }}
    +

    {{ value.description }}

    +
    + {{ form_it_delete.csrf_token }} + {{ form_it_delete.index(value=loop.index) }} + +
    +
    • + {% endfor %} +
+
+
+ + + + +
+
+ +
+

+ Item creation +

+
+ {{ form_it_new.hidden_tag() }} + {{ form_it_new.it_new_title.label.text }} + {{ form_it_new.it_new_title(class="w-full input validator mt-1 mb-4") }} + + {{ form_it_new.description.label.text }} + {{ form_it_new.description(class="w-full input validator mt-1 mb-2") }} + {{ form_it_new.price.label.text }} + {{ form_it_new.price(class="w-full input validator mt-1 mb-2", pattern="(?:€|$)?\d*(?:,|.)\d*") }} + {{ form_it_new.url.label.text }} + {{ form_it_new.url(class="w-full input validator mt-1 mb-2") }} + {{ form_it_new.image.label.text }} + {{ form_it_new.image(class="w-full input validator mt-1 mb-2") }} +
Please make sure that both inputs are filled and are valid.
+
+ + {{ form_it_new.it_new_submit(class="btn btn-soft btn-success + join-item sm:w-full") }} +
+
+
+ + +{% endblock content %} diff --git a/app/templates/index.html b/app/templates/index.html new file mode 100644 index 0000000..52b9159 --- /dev/null +++ b/app/templates/index.html @@ -0,0 +1,74 @@ +{% extends "base.html" %} + +{% block aboveNavbar %} + +
+
+
+
+

Hello there

+

+ Welcome to WishThat. The place to create and share wishlists online. +

+ +
+
+
+ +{% endblock aboveNavbar %} + +{% block content %} +
+

Why Use Wishthat?

+
+
+
+

Avoid Duplicate Purchases

+

Our system helps you keep track of what has been purchased, ensuring that you don't buy the same item twice.

+
+
+
+
+

No Account Needed

+

Start creating wishlists immediately without the hassle of signing up or logging in.

+
+
+
+
+

Price and Image Scraping

+

We scrape websites for the latest prices and images, making sure you don't have to painstakingly create wish items manually!

+
+
+
+
+

Easy to Use

+

Our user-friendly interface allows you to create and manage your wishlists effortlessly.

+
+
+
+
+

Fully Open Source

+

Wishthat is fully open-source and FOSS, allowing anyone to check our security.

+
+
+
+
+

Mobile friendly design

+

The software has been designed with mobile friendlyness in mind. Meaning that your phone addicted friends may enjoy this site as well.

+
+
+
+ + + +
+
+

© 2025 jsw. All rights reserved.

+
+
+
+ +{% endblock content %} diff --git a/app/templates/new.html b/app/templates/new.html index 1979102..f04c1a6 100644 --- a/app/templates/new.html +++ b/app/templates/new.html @@ -1,11 +1,22 @@ -
- {{ form.hidden_tag() }} +{% extends "base.html" %} - {{ form.title.label }} - {{ form.title() }} +{% block content %} - {{ form.description.label }} - {{ form.description() }} +
+ + {{ form.hidden_tag() }} - {{ form.submit() }} - +

New wishlist

+ + {{ form.title.label.text }} + {{ form.title(class="w-full input validator mt-1 mb-4", placeholder="Wishlist Title") }} + + {{ form.description.label.text }} + {{ form.description(class="w-full textarea validator mt-1 mb-2", placeholder="Wishlist Description") }} +
Please make sure that both inputs are filled.
+ + {{ form.submit(class="btn btn-soft btn-success w-full") }} + +
+ +{% endblock content %} diff --git a/app/templates/post_new.html b/app/templates/post_new.html new file mode 100644 index 0000000..5c7ff1c --- /dev/null +++ b/app/templates/post_new.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} + +{% block content %} + + +
+

Before you continue

+

Wishthat operates using two distinct passcodes: one for viewing the wishlist and another for editing it. Think of it as a combination of a username and password. Please navigate to the edit page and bookmark the URL, as you will not be able to edit your wishlist if you lose this passcode. You can also find the passcode for viewing the wishlist on the edit page.

+
+ View + Edit +
+
+ + +{% endblock content %} diff --git a/app/templates/view.html b/app/templates/view.html index 5df40ec..cb0e3ba 100644 --- a/app/templates/view.html +++ b/app/templates/view.html @@ -1,46 +1,76 @@ -

{{wishlist.title}}

-{{wishlist.description}} +{% extends "base.html" %} +{% block content %} +
+ +
+

{{wishlist.title}}

+

{{wishlist.description}}

+
-edit +
+ {% for item in wishlist.items %} +
+
+ {{ item.title }} +
+
+

{{ item.title }}

+

{{ item.description }}

+
+
{{ "€" ~ item.price if not item.bought }} +
+ + {{ form.csrf_token }} + {{ form.num(value=loop.index) }} +
+
+
+
+ {% endfor %} +
+
- - - -

Are you sure you bought this product. You won't be able to undo this.

-
- - -
+ + + + +
+

Still empty!

+

Hi there. It seems like you are here a bit too early, the wishlist is still empty. Please come back later.

+
+
+ Go back +
+
+
+{% endblock content %} diff --git a/app/views.py b/app/views.py index c168c6a..fabb07e 100644 --- a/app/views.py +++ b/app/views.py @@ -1,4 +1,5 @@ -from flask import url_for, redirect, render_template, abort +import json +from flask import request, url_for, redirect, render_template, abort from app import app, db from app.forms import ( NewWishlist, @@ -12,26 +13,36 @@ from app.forms import ( ) from app.models import Wishlist, Item from uuid import UUID, uuid4 as uuid +from app.scrapers import scrapeSite @app.route("/") def index(): - return "hello" + return render_template("index.html") @app.route("/new", methods=["GET", "POST"]) def new(): form = NewWishlist() + if form.validate_on_submit(): wishlist = Wishlist(str(form.title.data), str(form.description.data)) db.session.add(wishlist) db.session.commit() - return redirect(url_for("view", id=wishlist.viewId)) + return redirect( + url_for("postNew", viewId=wishlist.viewId, editId=wishlist.editId) + ) return render_template("new.html", form=form) +@app.route("/post_new//") +def postNew(viewId: str, editId: str): + + return render_template("post_new.html", viewId=viewId, editId=editId) + + @app.route("/edit/", methods=["GET", "POST"]) def edit(id: str): wishlist: Wishlist = db.one_or_404( @@ -54,8 +65,13 @@ def edit(id: str): return redirect(url_for("index")) elif form_wl_editinfo.validate_on_submit() and form_wl_editinfo.wl_edit_submit.data: - wishlist.title = str(form_wl_editinfo.title.data) - wishlist.description = str(form_wl_editinfo.description.data) + if form_wl_editinfo.title.data != "" and form_wl_editinfo.title.data != None: + wishlist.title = str(form_wl_editinfo.title.data) + if ( + form_wl_editinfo.description.data != "" + and form_wl_editinfo.description.data != None + ): + wishlist.description = str(form_wl_editinfo.description.data) db.session.commit() return redirect(url_for("edit", id=id)) @@ -69,12 +85,13 @@ def edit(id: str): return redirect(url_for("edit", id=wishlist.editId)) elif form_it_new.validate_on_submit() and form_it_new.it_new_submit.data: + print("NEW") f = form_it_new price = f.price.data if f.price.data != None else 0 item = Item( str( - f.title.data, + f.it_new_title.data, ), str( f.description.data, @@ -91,7 +108,7 @@ def edit(id: str): db.session.commit() return redirect(url_for("edit", id=id)) - elif form_it_delete.validate_on_submit() and form_it_delete.it_del_submit.data: + elif form_it_delete.validate_on_submit() and form_it_delete.index.data: index = parseHiddenIndex(form_it_delete.index, wishlist.items) if index == None: return abort(400) @@ -118,6 +135,7 @@ def view(id: str): db.select(Wishlist).filter_by(viewId=UUID(id)), description="Failed to get wishlist. Are you sure this is the correct url?", ) + checkform = CheckItem() checkform.num if checkform.validate_on_submit(): @@ -131,3 +149,16 @@ def view(id: str): return redirect(url_for("view", id=id)) return render_template("view.html", wishlist=wishlist, form=checkform) + + +@app.route("/scrape", methods=["GET"]) +def scrape(): + url = request.args.get("url") + if url is None: + abort(400) + + scraped = scrapeSite(url) + if scraped is None: + abort(404) + + return json.dumps(scraped.__dict__) diff --git a/flake.nix b/flake.nix index 6fa056c..029325b 100644 --- a/flake.nix +++ b/flake.nix @@ -14,7 +14,7 @@ { nativeBuildInputs = [ (pkgs.python3.withPackages - (x: [x.flask x.flask-wtf x.wtforms x.flask-sqlalchemy])) + (x: [x.flask x.flask-wtf x.wtforms x.flask-sqlalchemy x.beautifulsoup4 x.types-beautifulsoup4 x.requests])) pkgs.entr ]; }; diff --git a/instance/application.db b/instance/application.db index f0c5b91..b7835f8 100644 Binary files a/instance/application.db and b/instance/application.db differ