From e986ed8d033fbcf4ef77031084fe67d565c3abe4 Mon Sep 17 00:00:00 2001 From: cevoj35548 Date: Fri, 28 Apr 2023 14:45:09 +0000 Subject: [PATCH] Much faster performance, fix csv export, add "Final URL" column, bump to 1.13 --- FmhyChecker.pyw | 72 +++++++++++++++++++++++++++--------------- MainWindow.ui | 84 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 26 deletions(-) diff --git a/FmhyChecker.pyw b/FmhyChecker.pyw index 0f1cc25..c4b10ac 100644 --- a/FmhyChecker.pyw +++ b/FmhyChecker.pyw @@ -17,29 +17,9 @@ import darkdetect import ctypes as ct -# regex for scraping links from wikis (links must include https) -url_regex = re.compile(r'(?:https?|ftp|file):\/\/(?:ww(?:w|\d+)\.)?((?:[\w_-]+(?:\.[\w_-]+)+)[\w.,@?^=%&:\/~+#-]*[\w@?^=%&~+-])') -# regex for scraping lists of urls (NOT including https) -list_regex = re.compile(r'^[\w]*\.[\w.,@?^=%&:\/~+#-]*[\w@?^=%&~+-]', re.MULTILINE) -# urls to scrape (feel free to add more!) -URLS = { - 'https://gitlab.com/nbatman_/deleted-links/-/raw/main/deleted-links': list_regex, - 'https://raw.githubusercontent.com/nbats/FMHYedit/main/single-page': url_regex, -} - # fake headers headers = Headers(headers=True) -# scrape wiki -elapsed = time.perf_counter() - -resps = grequests.map([grequests.get(l) for l in URLS], size=len(URLS)) -wiki = set() -for resp, reg in zip(resps, URLS.values()): - wiki.update(set(re.findall(reg, resp.text))) - -print(f'Wiki scraped in {time.perf_counter() - elapsed:0.4f} sec. Found {len(wiki)} links.') - def resource_path(relative_path): try: @@ -85,6 +65,7 @@ class UI(QMainWindow): self._highlight_col = QtGui.QColor(255, 128, 0) self.checkSelected.setVisible(False) + self.progressBarFrame.setVisible(False) self.outputTree.header().setSectionsMovable(False) self.outputTree.setColumnWidth(0, 50) self.outputTree.setColumnWidth(1, 180) @@ -138,13 +119,14 @@ class UI(QMainWindow): try: with open(file_path, 'w', newline='') as csvfile: writer = csv.writer(csvfile, dialect='excel', quoting=csv.QUOTE_MINIMAL) - writer.writerow(['Request URL', 'Unique?', '# Redirects', 'Status', 'Reason']) + writer.writerow(['Request URL', 'Final URL', 'Unique?', '# Redirects', 'Status', 'Reason']) for link in links: full_link = ''.join(link) if full_link in self.tested_items: if type(self.tested_items[full_link]) is str: redirects = status_code = '' reason = self.tested_items[full_link] + final_url = '' else: redirects = str(len(self.tested_items[full_link].history)) reason = self.tested_items[full_link].reason @@ -154,10 +136,13 @@ class UI(QMainWindow): *self.tested_items[full_link].history, self.tested_items[full_link]) )+')' + final_url = self.tested_items[full_link].url else: - reason = redirects = status_code = '' + reason = redirects = final_url = status_code = '' writer.writerow([ - full_link, + re.sub(r'[^\x00-\x7F]+', '?', full_link), # remove non ascii characters + final_url, + redirects, 'FALSE' if link[1] in wiki else 'TRUE', redirects, status_code, @@ -252,6 +237,11 @@ class UI(QMainWindow): links = re.findall(self.group_url_regex, text) self.valid_links, self.duped_links, self.tested_links = [], [], [] + # enable progress bar + self.progressBar.setMaximum(len(links)) + self.progressBar.setValue(0) + self.progressBarFrame.setVisible(True) + self.outputTree.setVisible(False) # hide the treeview while updating for performance # populate tree for n, link in enumerate(links): if self._new_event: @@ -261,8 +251,9 @@ class UI(QMainWindow): item = QtWidgets.QTreeWidgetItem(self.outputTree) full_link = ''.join(link) item.setText(1, full_link) - if len(links) > 100 and not n % 10: + if len(links) > 100 and not n % 100: # process in chunks to allow for UI updates + self.progressBar.setValue(n) QtWidgets.QApplication.processEvents() with suppress(RuntimeError): if full_link in self.tested_items: @@ -277,6 +268,9 @@ class UI(QMainWindow): else: item.setText(0, "\u2705") self.valid_links.append(full_link) + # show finished tree + self.outputTree.setVisible(True) + self.progressBarFrame.setVisible(False) # toggle buttons self.copyValid.setEnabled(bool(self.valid_links)) self.copyDupes.setEnabled(bool(self.duped_links)) @@ -288,7 +282,7 @@ class UI(QMainWindow): def retranslateUi(self): # Set text (with translations) _translate = QtCore.QCoreApplication.translate - self.setWindowTitle(_translate("MainWindow", "Dupe Checker v1.12")) + self.setWindowTitle(_translate("MainWindow", "Dupe Checker v1.13")) self.label.setText(_translate("MainWindow", "FMHY Dupe Tester")) self.label_2.setText(_translate("MainWindow", "by cevoj35548")) self._placeholderText = _translate("MainWindow", "Paste a list of links here...") @@ -341,6 +335,34 @@ def dark_palette(): if __name__ == "__main__": import sys app = QtWidgets.QApplication(sys.argv) + + # regex for scraping links from wikis (links must include https) + url_regex = re.compile(r'(?:https?|ftp|file):\/\/(?:ww(?:w|\d+)\.)?((?:[\w_-]+(?:\.[\w_-]+)+)[\w.,@?^=%&:\/~+#-]*[\w@?^=%&~+-])') + # regex for scraping lists of urls (NOT including https) + list_regex = re.compile(r'^[\w]*\.[\w.,@?^=%&:\/~+#-]*[\w@?^=%&~+-]', re.MULTILINE) + # urls to scrape (feel free to add more!) + URLS = { + 'https://gitlab.com/nbatman_/deleted-links/-/raw/main/deleted-links': list_regex, + 'https://raw.githubusercontent.com/nbats/FMHYedit/main/single-page': url_regex, + } + # scrape wiki + elapsed = time.perf_counter() + try: + resps = grequests.map([grequests.get(l) for l in URLS], size=len(URLS)) + except ConnectionError: + # make qt message box + msg = QtWidgets.QMessageBox() + msg.setIcon(QtWidgets.QMessageBox.Critical) + msg.setText("Could not connect to the internet. Please check your connection and try again.") + msg.setWindowTitle("Connection Error") + msg.exec_() + exit(1) + wiki = set() + for resp, reg in zip(resps, URLS.values()): + wiki.update(set(re.findall(reg, resp.text))) + + print(f'Wiki scraped in {time.perf_counter() - elapsed:0.4f} sec. Found {len(wiki)} links.') + fonts_dir = resource_path('assets/fonts') for f in os.listdir(fonts_dir): QtGui.QFontDatabase.addApplicationFont(os.path.join(fonts_dir, f)) diff --git a/MainWindow.ui b/MainWindow.ui index 15042ec..03af14f 100644 --- a/MainWindow.ui +++ b/MainWindow.ui @@ -107,7 +107,7 @@ 4 - + 0 @@ -243,6 +243,88 @@ + + + + + 4 + + + 0 + + + 0 + + + 0 + + + 0 + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + Segoe UI Variable Display + 11 + + + + Processing... + + + Qt::AlignCenter + + + + + + + true + + + + Segoe UI Variable Display + 11 + + + + 0 + + + %p% (%v/%m) + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + +