I want use a parallel downloading videos from youtube, but my code ending with exception "PicklingError". Can you help guys with code, how it should be, please.
Another fixed variant:
import sys
#from pathos.multiprocessing import ProcessingPool as Pool
from multiprocessing import Pool
from pytube import YouTube
from youtubeMultiDownloader import UiMainWindow
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QFileDialog
class YouTubeInstance:
def __init__(self, path):
self.youtube = YouTube
self.path = path
#self.ui_obj = ui_obj
def download_file(self, url):
self.youtube(url).streams.get_highest_resolution().download(self.path)
#self.ui.ui.youtube_outputs.setText(f'Video \'{self.youtube.title}\' has been downloaded successfully!')
class YouTubeMultiDownloader(QtWidgets.QMainWindow):
def __init__(self):
super().__init__()
self.pool = Pool
self.ui = UiMainWindow()
self.ui.setup_ui(self)
self.path_to_dir = None
self.urls = None
def _get_urls_from_form(self):
self.urls = self.ui.youtube_urls.toPlainText().split('\n')
return len(self.urls)
def choose_directory(self):
self.path_to_dir = str(QFileDialog.getExistingDirectory(self, "Select Directory"))
def run_multi_downloads(self):
youtube = YouTubeInstance(self.path_to_dir)
self.pool(self._get_urls_from_form()).map(youtube.download_file, self.urls)
if __name__ == "__main__":
app = QtWidgets.QApplication([])
application = YouTubeMultiDownloader()
application.show()
sys.exit(app.exec_())
Updated:
My ui :)
Error 1 fixed:
Error 2 fixed:
Error 3 actual:
You've got the wrong side of the stick. Take a look at multiprocessing module documents. As it says, calling Pool method is for running multiple instance of same function simultaneously (in parallel). So call Pool method as many numbers you want, meanwhile your method does not any parameters, call it without any arguments:
with Pool(5) as p:
print(p.map(YouTubeMultiDownloader))
It create 5 parallel instance. You can change the code an refine your errors.
Related
First of all, I'm currently migrating my source code from PyQt5 to PySide2 which requires me to change some of the syntaxes. As this site said that it only needs 3 things to do migrate from PyQt to Pyside2.
1.app.exec_. exec_ was used as exec is a Python2 keyword. Under Python3, PyQt5 allows the use of exec but not PySide2.
2.Under PyQt5 it’s QtCore.pyqtSignal and QtCore.pyqtSlot and under PySide2 it’s QtCore.Signal and QtCore.Slot .
3.loading Ui files.
But anyway later on when I tried to run my code it gave me this following error:
QThread: Destroyed while thread is still running
I had more than 2000 lines of code and I cannot determine which is the cause of this other than my last action which is trying to call QFileDialog which shouldn't be a problem (I've tested this with PyQt import and there's no problem and no warning at all). But in PySide2 it definitely might be the cause of it. I look up into this, he doesn't have the same problem as mine exactly. I'm not trying to call QFileDialog from different thread.
this is the minimal reproducible example of my working code in PyQt5:
import sys
import os
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QMainWindow, QFileDialog, QMessageBox, QWidget, QDialog
import random
class MyWidget(QtWidgets.QWidget):
def __init__(self):
QtWidgets.QWidget.__init__(self)
self.path = os.path.abspath(os.path.dirname(sys.argv[0]))
self.button = QtWidgets.QPushButton("Open File")
self.labelFile = QtWidgets.QLabel("empty")
self.labelData = QtWidgets.QLabel("None")
self.layout = QtWidgets.QVBoxLayout()
self.layout.addWidget(self.button)
self.layout.addWidget(self.labelFile)
self.layout.addWidget(self.labelData)
self.setLayout(self.layout)
self.button.clicked.connect(self.open_file)
timer = QtCore.QTimer(self)
timer.timeout.connect(self.update_data_value)
timer.start(1000)
def open_file(self):
x = QFileDialog.getOpenFileName(self,"Pilih File CSV yang Ingin Diproses",self.path,"CSV Files (*.csv)")
self.labelFile.setText(x[0])
def update_data_value(self):
self.DataProcess = DataProcess()
self.DataProcess.progress.connect(self.update_data_label)
self.DataProcess.start()
def update_data_label(self,x):
self.labelData.setText(str(x[0]))
class DataProcess(QtCore.QThread):
progress = QtCore.pyqtSignal(object)
def __init__(self):
QtCore.QThread.__init__(self)
def run(self):
x = random.randint(1,100)
self.progress.emit([str(x)+ " from thread"])
if __name__ == "__main__":
app = QtWidgets.QApplication(sys.argv)
widget = MyWidget()
widget.show()
sys.exit(app.exec_())
and this is the non-working one in PySide2 after renaming import accordingly to PySide2 also renaming 'pyqtsignal' to 'Signal'
import sys
import os
from PySide2 import QtCore, QtGui, QtWidgets
from PySide2.QtWidgets import QMainWindow, QFileDialog, QMessageBox, QWidget, QDialog
import random
class MyWidget(QtWidgets.QWidget):
def __init__(self):
QtWidgets.QWidget.__init__(self)
self.path = os.path.abspath(os.path.dirname(sys.argv[0]))
self.button = QtWidgets.QPushButton("Open File")
self.labelFile = QtWidgets.QLabel("empty")
self.labelData = QtWidgets.QLabel("None")
self.layout = QtWidgets.QVBoxLayout()
self.layout.addWidget(self.button)
self.layout.addWidget(self.labelFile)
self.layout.addWidget(self.labelData)
self.setLayout(self.layout)
self.button.clicked.connect(self.open_file)
timer = QtCore.QTimer(self)
timer.timeout.connect(self.update_data_value)
timer.start(1000)
def open_file(self):
x = QFileDialog.getOpenFileName(self,"Pilih File CSV yang Ingin Diproses",self.path,"CSV Files (*.csv)")
self.labelFile.setText(x[0])
def update_data_value(self):
self.DataProcess = DataProcess()
self.DataProcess.progress.connect(self.update_data_label)
self.DataProcess.start()
def update_data_label(self,x):
self.labelData.setText(str(x[0]))
class DataProcess(QtCore.QThread):
progress = QtCore.Signal(object)
def __init__(self):
QtCore.QThread.__init__(self)
def run(self):
x = random.randint(1,100)
self.progress.emit([str(x)+ " from thread"])
if __name__ == "__main__":
app = QtWidgets.QApplication(sys.argv)
widget = MyWidget()
widget.show()
sys.exit(app.exec_())
so after creating this minimal example, I realized that PySide QFileDialog makes the QThread stop while PyQt QFileDialog doesn't freeze the main thread. Is there anything I could do to handle this in similar syntax architecture? (e.g not using "movetothread" or "QObject")
The problem is that you're overwriting self.DataProcess every time a new thread is created, which may cause the previous object to be garbage-collected by Python before Qt has a chance to delete it. This can result in a core-dump if Qt tries to delete an object which is no longer there. Problems of this kind are quite common in both PyQt and PySide, and are usually caused by not keeping proper references to dependant objects. The normal solution is to ensure that the affected objects are given a parent and, if necessary, explicitly delete them at an appropriate time.
Here is one way to fix your example:
class MyWidget(QtWidgets.QWidget):
...
def update_data_value(self):
# ensure the thread object has a parent
process = DataProcess(self)
process.progress.connect(self.update_data_label)
process.start()
def update_data_label(self,x):
self.labelData.setText(str(x[0]))
class DataProcess(QtCore.QThread):
progress = QtCore.Signal(object)
def __init__(self, parent):
# ensure the thread object has a parent
QtCore.QThread.__init__(self, parent)
def run(self):
x = random.randint(1,100)
self.progress.emit([str(x)+ " from thread"])
# explicitly schedule for deletion
self.deleteLater()
It's hard to say exactly why PySide behaves differently to PyQt in this particular case. It usually just comes down to low-level differences between the two implementations. Probably there are equivalent cases that affect PyQt but not PySide. However, if you manage object references and cleanup carefully, such differences can usually be eliminated.
I am new to Python and I think I broke my python :(
I was trying Sentdex's PyQt4 YouTube tutorial right here.
I made the changes from PyQt4 to PyQt5. This is the code I was playing around. So I think, I messed up by printing the whole page on the console.
Now the output is:
Load finished
Look at you shinin!
Press any key to continue . . .
This is being shown for any code executed. That is python shows this code even if I try print("hello") in Visual code. I even tried to restart. Now like a virus, it is not clearing.
import bs4 as bs
import sys
import urllib.request
from PyQt5.QtWebEngineWidgets import QWebEnginePage
from PyQt5.QtWidgets import QApplication
from PyQt5.QtCore import QUrl
class Page(QWebEnginePage):
def __init__(self, url):
self.app = QApplication(sys.argv)
QWebEnginePage.__init__(self)
self.html = ''
self.loadFinished.connect(self._on_load_finished)
self.load(QUrl(url))
self.app.exec_()
def _on_load_finished(self):
self.html = self.toHtml(self.Callable)
print('Load finished')
def Callable(self, html_str):
self.html = html_str
self.app.quit()
def main():
page = Page('https://pythonprogramming.net/parsememcparseface/')
soup = bs.BeautifulSoup(page.html, 'html.parser')
js_test = soup.find('p', class_='jstest')
print js_test.text
print (soup)
#js_test = soup.find('div', class_='aqi-meter-panel')
#display.popen.terminate()
if __name__ == '__main__': main()
OK, so finally got the problem fixed..went manually inside the temp files in C:\Users\xxx\AppData\Local and started on a deletion rampage...removed many files and folder remotely related to python,vscode and conda...this gave an error warning first time I executed my program again...then on subsequent run...no issue...python back to its normal self...surprised that I was not able to find any solution on the net for this.
When i run python-pyside2 project server first, then it works well.
And the site also work well, but if i press F5 btn to refresh in browser.
Site shows error page Runtime at/
import sys
from urllib.request import urlopen
from bs4 import BeautifulSoup
from PySide2.QtGui import *
from PySide2.QtCore import *
from PySide2.QtWebKitWidgets import *
from PySide2.QtWidgets import QApplication
class dynamic_render(QWebPage):
def __init__(self, url):
self.frame = None
self.app = QApplication(sys.argv)
QWebPage.__init__(self)
self.loadFinished.connect(self._loadFinished)
self.mainFrame().load(QUrl(url))
QTimer.singleShot(0, self.sendKbdEvent)
QTimer.singleShot(100, app.quit)
self.app.exec_()
def _loadFinished(self, result):
self.frame = self.mainFrame()
self.app.quit()
self.app = None
Below, scaping code using pyside2:
I don't know how can i fix it?
Best regards.
Thanks.
Check if already an instance of QApplication is present or not as the error occurs when an instance is already running and you are trying to create a new one.
Write this in your main function:
if not QtWidgets.QApplication.instance():
app = QtWidgets.QApplication(sys.argv)
else:
app = QtWidgets.QApplication.instance()
For my pyside2 unit test the following worked fairly well
import PySide2
import unittest
class Test_qapplication(unittest.TestCase):
def setUp(self):
super(Test_qapplication, self).setUp()
if isinstance(PySide2.QtGui.qApp, type(None)):
self.app = PySide2.QtWidgets.QApplication([])
else:
self.app = PySide2.QtGui.qApp
def tearDown(self):
del self.app
return super(Test_qapplication, self).tearDown()
it was loosely based on stack overflow : unit-and-functional-testing-a-pyside-based-application
Trying to implement multiprocessing within a PyQt5 GUI framework to process multiple files in a parallel process, preferably in a QThread() background. Running the code below seems to create the multiple processes but has additional issue that multiple GUI windows appear and everything seems locked out at that point.
import multiprocess as mp
from PyQt5.QtWidgets import QMainWindow
class MyApp(QMainWindow, myUiMainWindow):
def __init__(self):
super(self.__class__, self).__init()
self.setupUI(self)
self.pushButton.clicked.connect(self.doMyStuff)
def consumer(self, inQ, outQ):
val = inQ.get()
ret = self.process_single(val)
outQ.put(ret)
def process_single(self, f):
<process each file f>
<update progress bar>
return f
def doMyStuff(self):
<get file_list from GUI Widget>
n_w = len(file_list) if len(file_list) < 5 else 5
inQ = mp.Queue()
outQ = mp.Queue()
workers = [mp.Process(target=consumer, args=(inQ, outQ) for i in range(n_w)]
[w.start() for w in workers]
[inQ.put(f) for f in file_list]
[inQ.put(None) for i in range(n_w)]
completed_files = []
while len(completed_files) != len(file_list):
completed_files.append(outQ.get())
[w.join() for w in workers]
Guess what...
implement the following to build the GUI only once:
import multiprocess as mp
from PyQt5 import QtWidgets
from QtWidgets import QMainWindow
... snippet ... your script code ...
if __name__ == '__main__':
app = QtWidgets.QApplication(sys.argv)
window = MyApp()
window.setGeometry(300, 100, 600, 600) # adjustable (left,top,width,height)
window.show()
sys.exit(app.exec_())
... you probably figured this one out by now ;p
Ok so maybe it isnt a bug, but I cant get it to work.
say you have to classes that both use PyQt4. One is called Audio.py and uses Phonon to play a sound file. the other is called GUI.py and uses QtGui to display a screen. GUI needs to be able to call and use Audio.py whenever it wants (import Audio). It will import and send the call to my Audio class but because Audio is not started (double click to run) it does not the code (app=QApplication(sys.argv); sys.exit(app.exec_())). so while the Audio class runs when it is run, when you import it, it will not play sounds because its own QApplication loop has not been started.
Any Help?
Edit:
Added class Engine
these are 2 seperate python files (.py)
import Library,Player,sys
from PyQt4.QtGui import QApplication
class Engine(object):
def __init__(self,path,song=None):
self.counter=0
self.path=path
self.lib=Library.Library(self.path)
if song is None:
self.player=Player.Player(self.lib.getSong(self.counter))
else:
self.player=Player.Player(path+song)
def updatePlayer(self,songStatus):
self.player.findStatus(songStatus)
def getCurrentSong(self):
return self.lib.getSong(self.counter)
if __name__=='__main__':
app=QApplication(sys.argv)
e=Engine('D:/Music/','Yeah!.mp3')
e.updatePlayer('Play')
sys.exit(app.exec_())
import sys
from PyQt4.QtGui import QApplication
from PyQt4.QtCore import QObject
from PyQt4.phonon import Phonon
class Player(QObject):
def __init__(self,song):
super(QObject,self).__init__()
self.song=song
self.media=None
#self.metaInfo=Phonon.MediaObject(self)
#self.metaInfo.currentSourceChanged.connect(self.disMetaData)
self.initMedia()
self.findStatus()
def initMedia(self):
if not self.media:
self.media=Phonon.MediaObject()
audioOutput=Phonon.AudioOutput(Phonon.MusicCategory,self)
Phonon.createPath(self.media,audioOutput)
self.media.setCurrentSource(Phonon.MediaSource(self.song))
def findStatus(self,status=None):
if status is not None:
if status=='Play':
self.playSong()
return
if status=='Stop':
self.stopSong()
return
if status=='Pause':
self.pauseSong()
return
if status=='Next':
nextSong()
return
if status=='Previous':
self.previousSong()
return
def playSong(self):
self.media.play()
def stopSong(self):
self.media.stop()
def pauseSong(self):
self.media.pause()
def nextSong(self):
'''nextSong code'''
def previousSong(self):
'''previousSong code'''
if __name__=='__main__':
app=QApplication(sys.argv)
p=Player('D:/Music/Yeah!.mp3')
p.findStatus('Play')
sys.exit(app.exec_())
To be sure you make actions when the event loop is already running use QTimer::singleShot:
QtCore.QTimer.singleShot(0, some_function)