From c51464d68e9682c7738fb5f24888c1d0589eebb6 Mon Sep 17 00:00:00 2001
From: Your Name <you@example.com>
Date: Mon, 27 Oct 2025 03:03:32 -0700
Subject: [PATCH] MY ASS BROKEN!!!!

---
 __pycache__/test1.cpython-314.pyc      | Bin 0 -> 482 bytes
 wcr/__pycache__/config.cpython-313.pyc | Bin 0 -> 1594 bytes
 wcr/__pycache__/config.cpython-314.pyc | Bin 1107 -> 1615 bytes
 wcr/config.py                          |   6 ++-
 wcr/crawl_wcr.py                       |  71 +++++++++++++------------
 5 files changed, 42 insertions(+), 35 deletions(-)
 create mode 100644 __pycache__/test1.cpython-314.pyc
 create mode 100644 wcr/__pycache__/config.cpython-313.pyc

diff --git a/__pycache__/test1.cpython-314.pyc b/__pycache__/test1.cpython-314.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f3c1e76bd5b33b8057ba44611e866dd87fc781dc
GIT binary patch
literal 482
zcmYjNJxc>Y5S_h`n8f%2DMV3>1d&q^6cGuMMhgWw!hw+D;&HibK1^~OcQ;5<r4UKu
zU$FPzScu>uSXkLb^8<AEE}{eT-s~*z%`jWDGf5C}xBp$g7yK?CzoAZwab6SyxBv^C
z0*@GRWGM^4m6B?y2StF&G{7>Hv^B7_JrpTdg(4fKp?}Dh=|vwAN`IN-T>%5ICn^97
z{v7^w(O=h6vS6rzUdOE#33R_vtGT9u0=+{UK7W9*oDk%Wa`rjHl-Vb}3MIue%Ra`e
z$w|j9^mv{4nF1x3*yGIhsC(VQ)aID+o$XF9h*b$GyOhfF$R*jlfC0Q}iQ&eL8Pcsu
z<8n;gaoTRfcbxu;Tm;ihwd*wnl^fivXI#cI;ev)9#3{b&iV-YO2n)TX+IQR7acE4K
z(J2YimL@{)!#1gOTR2bW#8X~~tqT~b2%$Gfe`yLzk5ZtehW#gX@kL8KCXHup^HT+F
HDYW?kmHBI%

literal 0
HcmV?d00001

diff --git a/wcr/__pycache__/config.cpython-313.pyc b/wcr/__pycache__/config.cpython-313.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7b7cdaa7e1c100e6ef5adc23d0dd00ea330cd846
GIT binary patch
literal 1594
zcma)6&u<$=6n^{5cIw(8aY2<)lT8yfED=$OTM$u5lr*VQ+PIOomLkQHwb>mz24~02
ztdoXQsE}wS)FPyaUXb8KIPxdZ69*0siBuQ`Apw`(BFupw2Y9ow9Y9FoNjvYGdEd<Y
zc4po*6NyuR_SM9ntDma?Kgv$qupe4;9Njk9U@O;wDWO-HieA00y{0qWgo1t^F1P3=
z*jgHxVe8mN%8F$?k&&#Wh<KGcUeTig@z>oJJ#<lQHT;?-MW*V5<xVu$meCmjIFC{;
zBU74xGFR<zl@r(1I$ZTcuHNBl%leTTfLx@Fj(x7{7`M@qvNUKLujsy>*N0#LPFNv4
zruN(;9|h+u{=c!DXph^Gype=V=YLJ;$7A!2X`|foC8Y$fYW+}BtY}t5XJ?A@H|<*?
z$lG(d5G+pHb_lZ53v-q(^i|3Oq1QQKB35%a-=NGBG0zE{6^E1hIZXe<NAH<eol2E>
zW<X7s`aW?331?@<O)Dx?%0;A1<iQJLZIi!Q6<sYBWg^;w!kA+W`$mmb{J@HcsJBuK
zvr7;PkU(k(L%y>{gzBvbjWEXL#+#vusVn1!JnH3z`Av_}r5m}zEa7hiv{qQ$3|6Uc
zE;9N$O5_EXIU7~N3L7pfpo~(bJX+fnJ*c&asanD+)MwA)@-hX#kIfbws$E9+LGPIl
ztq-h+*Y<iZ+>QSK-cO0%2gQ5E$BCic#L&+0m&1+3P$S{pRS#ouI=OZ2*I%M=y63^v
zy{X4>b2n}_LB(XTMB}L!cHEtOBbnKcT{?&*|2Q--@H@wQ0C%QenjVKQ#`|V2YLCt)
zXGXL~BL-r(RVYXSFz^ct%|oyadC2`eDj8_U#bv^>WQS&}X~~x>{L9*vpX$5;?ZOxW
zOUu4>&Yrt5cjcDJX`Q*GSZ7u9YVPKOxnA)I<^Rwpvt{sWK99{Hn4!iL%wqx39y^r3
z4%>za3vqxr#>>jf8eV36k8ncvGq6;lj$RZRqZ?cfnI=o46I~N!;*;AoHYqOD7zrPu
zV?BIyqDoz-$|pxVyr*#*KZDH^JZ;?ZuLs5+4!-~5{dEMtaUBD9dQys|i72^E+<HKS
z#z`PzB@(!+PPK~fUBs<3P8pYb+|rTykK}9^<0<^O1Y7Wpe(K%q+u25HY+uhD#FM+p
z%*W4s+W$%a{=nE?GP4)YH1tdpseqN$9aht5>!MP|x^bMzCqYDxURowMHN3uIBGxCn
x&&ej-nb^b{_3BkJ$wqKYO679=P*W7;dq{r=ecwX-oy5<PxRPw9RpnAR_b<;&Rp<Z!

literal 0
HcmV?d00001

diff --git a/wcr/__pycache__/config.cpython-314.pyc b/wcr/__pycache__/config.cpython-314.pyc
index 05bd58ae77f0c35aeaf3ed2a877e2fbdb5bb418a..329c90a48c2a7e6d8d662383542353fab2f4070d 100644
GIT binary patch
delta 660
zcmZuu&ubGw6n?Y2Nz=yND4~r8%nu8xOC{)T^c3Pj+DkQtSUgEf+>EBO&9pPSF%=<E
z1i_1zTs-PYJof((Oi2&AC<Slc#CY)_`X;9S!UylYnfJbL-p89k<n<wa-q5E2?N#x6
zV`p1GMvB{W=b~Byyl|Dwf*IPmepeyqd;3Z<R@w3zteL5Bc8fZm=xOSQjzc7SY$41l
z`mm0n3s<4?XLk&$LSbA@!!?N>S(PLtUcT^;6~_0&aBw={qr8rC=lGR{iwhT)GlKct
zp{qW3GmF*BmolxjI%VQ7eR7(OKD02LfHZ1chD9A<^T-OEN{$C%YC|`ahNT$5B&0wt
z;;gzxOoc1hEWZR(36%WlK;<2FTgX~2!zDeKEZ(F|+TP%$Ah*G(%WC!2-@-(`=rX6~
zic<a;ZW?J~2E#{G(>Lthto3yE$(hHkN3B8D`YJSR-JebnQw=6)n>xNn165Ejh}@u_
z(`nRP*CavM@ws4J$j#<{N{RnxKA*#W99_sm7v5-756kz;{kXNK+3%vpu3<l$d69me
z-pg78!yZKKzGe>*3g$!*Ti+6!Zr)+dCVoHT9M{wPrXF1yy~<OF;wNPoZ6Z=!XLaAD
kB|eWesY}TECzTMg56SnCdJEA9`sZ+z7+;bKDU7&30A`qy-~a#s

delta 149
zcmX@lbD4uzn~#@^0SJz&{LfIF$ScWcHc{P-l~I{NlVxK@4U?L+vsFxJacWU<j7Mck
zQGSF^P>f4zadt_5L5y>LN@`3%Wl2VU-ef-JM@*V*ldrSt+O1^x3^K4t3P{}Iu*uC&
sDa}c>D^dV*8G*Q16i9qvW@Kc%&meV|LGC64cMI<qHf~0~B6*+;06+34mjD0&

diff --git a/wcr/config.py b/wcr/config.py
index 2244b8f..c628d6f 100644
--- a/wcr/config.py
+++ b/wcr/config.py
@@ -14,4 +14,8 @@ def get_db_connection():
     except mysql.connector.Error as err:
         print(f"❌ Failed to connect to DB: {err}")
         return  # don’t continue if DB failed
-    return db
\ No newline at end of file
+    return db
+
+def preload_source_urls(cursor):
+    cursor.execute("SELECT source_url FROM videos")
+    return set(row[0] for row in cursor.fetchall())
diff --git a/wcr/crawl_wcr.py b/wcr/crawl_wcr.py
index 713d66b..3c470ec 100644
--- a/wcr/crawl_wcr.py
+++ b/wcr/crawl_wcr.py
@@ -1,4 +1,4 @@
-from config import get_db_connection
+from config import get_db_connection, preload_source_urls
 import requests, time, mysql.connector
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin
@@ -6,36 +6,34 @@ from datetime import datetime
 
 
 def insert_video_to_db(data):
-    db = get_db_connection()
+    
     
     try:
         cursor = db.cursor()
         sql = """
-        INSERT IGNORE INTO videos (username, url, title, date, embed_link, source_url, created_at)
-        VALUES (%s, %s, %s, %s, %s, %s, NOW())
+        INSERT IGNORE INTO videos (username, date, embed_link, source_url, created_at)
+        VALUES (%s, %s, %s, %s, NOW())
                 """
 
         values = (
             data['username'],
-            data['url'],
-            data['title'],
             data['date'],
             data['embed_link'],
             data['source_url']
         )
         cursor.execute(sql, values)
         db.commit()
-        print("✅ Inserted into DB!")
+        if cursor.rowcount == 0:
+            print("❌ Video already exists in DB")
+        else:
+            print("✅ Inserted into DB!")
 
     except mysql.connector.Error as err:
         print(f"❌ Failed to insert: {err}")
 
-    finally:
-        cursor.close()
-        db.close()
+    
 
-def crawl_user_page(base_url, user_path):
-    full_url = urljoin(base_url, user_path)
+def crawl_user_page(full_url):
     response = requests.get(full_url)
     if response.status_code != 200:
         print(f"❌ Failed to load {full_url}")
@@ -43,13 +41,15 @@ def crawl_user_page(base_url, user_path):
 
     soup = BeautifulSoup(response.text, "html.parser")
 
+    data = parse_data(soup)
+    data["source_url"] = full_url
+    return data
     
 
 def parse_data(soup):
-    username = user_path.strip("/")
+    
     title_tag = soup.find("h1", class_="entry-title")
-    title = title_tag.text.strip() if title_tag else "(no title)"
-
+    username = title_tag.contents[0].strip() 
     date_tag = soup.find("span", class_="entry-date")
     date = date_tag.text.strip() if date_tag else None
 
@@ -75,7 +75,6 @@ def parse_data(soup):
 
         return {
             "username": username,
-            "title": title,
             "date": date,
             "embed_link": embed_link,
         }
@@ -94,14 +93,24 @@ def crawl_all(init_url):
             break
 
         soup = BeautifulSoup(response.text, "html.parser")
-        user_links = soup.find_all("a", class_="thumbnail-link", href=True)
-        if not user_links:
+        video_pages = soup.find_all("a", class_="thumbnail-link", href=True)
+        video_pages = [link['href'] for link in video_pages]
+        if not video_pages:
             print("⚠️ No user links found — reached end of site.")
             break
-
-        for link in user_links:
-            user_path = link["href"]
-            user_data = crawl_user_page(init_url, user_path)
+        cursor = db.cursor()    
+        seen_urls = preload_source_urls(cursor)
+        
+        for link in video_pages:
+            full_url = init_url + link
+            
+            if full_url in seen_urls:
+                print(f"⚠️ Skipping {link} - already seen.")
+                continue
+            
+            
+            
+            user_data = crawl_user_page(full_url) # slow as fuk
             if not user_data:
                 print("⚠️ Skipping empty user_data.")
                 continue
@@ -111,25 +120,19 @@ def crawl_all(init_url):
                 continue
 
             insert_video_to_db(user_data)
-            time.sleep(0.5)
+            
 
         page += 1
-        time.sleep(1)
+        
             
     print(f"\n✅ Finished crawling all pages. Total users: {len(all_data)}")
     return all_data
 
 if __name__ == "__main__":
+    db = get_db_connection()
     BASE_URL = "https://webcamrips.to"
     results = crawl_all(BASE_URL)
     print("💾 All data saved to users_data.json")
-            
-            
-   
-            
-                
-
-
-
-
-
+    cursor = db.cursor()
+    cursor.close()
+    db.close()
\ No newline at end of file