Do not process already present data - python-3.x

Question

I have a folder in which there are some video files. I want to extract frames from the videos but only those videos should be processed whose names are not present in the csv. It should check for the present video file names in the csv before processing the videos
def extractFrames(m):
global vid_name
vid_files=glob(m)
print(vid_files)
complete_videos = get_completed_videos()
print(complete_videos)
new_vid_files = [x for x in vid_files if get_vid_name(x) not in complete_videos]
for vid in new_vid_files:
print("path of video========>>>>.",vid)
v1=os.path.basename(vid)
try:
vid_name = get_vid_name(vid)
vidcap = cv2.VideoCapture(vid)
except cv2.error as e:
print(e)
except:
print('error')
#condition
fsize=os.stat(vid)
print('=============size of video ===================:' , fsize.st_size)
try:
if (fsize.st_size > 1000):
fps = vidcap.get(cv2.CAP_PROP_FPS) # OpenCV2 version 2 used "CV_CAP_PROP_FPS"
frameCount = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
duration = frameCount/fps
minutes = int(duration/60)
print('fps = ' + str(fps))
print('number of frames = ' + str(frameCount))
print('duration (S) = ' + str(duration))
if (duration > 1):
success,image = vidcap.read()
count=0
success=True
while success:
img_name = vid_name + '_f' + str(count) + ".jpg"
success,image = vidcap.read()
if count % 10 == 0 or count ==0:
target_non_target(img_name, image)
count+=1
vidcap.release()
cv2.destroyAllWindows()
except:
print("error")
print('finished processing video ',vid)
with open("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\"+'video_info.csv', 'a') as csv_file:
fieldnames = ['Video_Name','Process']
file_is_empty = os.stat("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\"+'video_info.csv').st_size == 0
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
if file_is_empty:
writer.writeheader()
writer.writerow({'Video_Name':vid_name,'Process':'done'})
def get_vid_name(vid):
return os.path.splitext(os.path.basename(vid))[0]
def get_completed_videos():
completed_videos = []
with open("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\video_info.csv") as csv_file:
for row in csv.reader(csv_file):
for col in range(0,len(row)):
try:
completed_videos.append(row[col])
except Exception as e:
print(str(e))
print(completed_videos[0])
return completed_videos
Suppose there are 3 videos in a folder. Code is successfully run for those 3 videos and their names are written in the csv. Now if i paste video number 4 in the folder then it should process only the 4th video after checking for the video names present in the csv. Currently it is repeatedly processing all the video files everytime the script is run.

Answer 1

First off, inside the for loop
v1=os.path.basename(vid_files[v_f])
Should be
v1=os.path.basename(new_vid_files[v_f])
Since you are looping over the new_vid_files range. Using those indices on the original list will give you unexpected items. Better yet, you can directly use a for-each loop (since you don't seem to be using v_f for anything other than list access) as follows:
for vid in new_vid_files:
And this vid would replace all instances of new_vid_files[v_f].
Next, you are using vid_name to write to the csv, so you need to perform the same operation for each item from vid_files before matching against complete_videos while creating the new_vid_files list.
If you create a method for getting the video name as follows:
def get_vid_name(vid_file):
return os.path.splitext(os.path.basename(vid_file))[0]
Then you can change the list comprehension to be
new_vid_files = [x for x in vid_files if get_vid_name(x) not in complete_videos]
Edit: As mentioned in the comments to the other answer, the output for complete_videos indicates it isn't being parsed properly. It is appending both the column headers and other unneeded columns. This code will work despite that, but it needs to be fixed. I am not solving it because it is a relatively simple change, and I want the OP to understand what they're doing wrong.

Answer 2

def extractFrames(m):
global vid_name
vid_files=glob(m)
print(vid_files)
complete_videos = get_completed_videos()
new_vid_files = [x for x in vid_files if get_vid_name(x) not in complete_videos]
for vid in new_vid_files:
print("path of video========>>>>.",vid)
v1=os.path.basename(vid)
try:
vid_name = get_vid_name(vid)
vidcap = cv2.VideoCapture(vid)
except cv2.error as e:
print(e)
except:
print('error')
#condition
fsize=os.stat(vid)
print('=============size of video ===================:' , fsize.st_size)
try:
if (fsize.st_size > 1000):
fps = vidcap.get(cv2.CAP_PROP_FPS) # OpenCV2 version 2 used "CV_CAP_PROP_FPS"
frameCount = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
duration = frameCount/fps
minutes = int(duration/60)
print('fps = ' + str(fps))
print('number of frames = ' + str(frameCount))
print('duration (S) = ' + str(duration))
if (duration > 1):
success,image = vidcap.read()
count=0
success=True
while success:
img_name = vid_name + '_f' + str(count) + ".jpg"
success,image = vidcap.read()
if count % 10 == 0 or count ==0:
target_non_target(img_name, image)
count+=1
vidcap.release()
cv2.destroyAllWindows()
except:
print("error")
print('finished processing video ',vid)
with open("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\"+'video_info.csv', 'a') as csv_file:
fieldnames = ['Video_Name','Process']
file_is_empty = os.stat("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\"+'video_info.csv').st_size == 0
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
if file_is_empty:
writer.writeheader()
writer.writerow({'Video_Name':vid,'Process':'done'})
def get_vid_name(vid):
return os.path.splitext(os.path.basename(vid))[0]
def get_completed_videos():
completed_videos = []
with open("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\video_info.csv") as csv_file:
for row in csv.reader(csv_file):
for col in range(0,len(row)):
try:
completed_videos.append(row[col])
except Exception as e:
print(str(e))
print(completed_videos[0])
return completed_videos