I have been trying to resize all images contained in nested directories and save the resulting image into directories with the same structure as the original one. I keep getting the error that the directory or file doesn´t exist (though it does really exist).
root_path= 'D:/Users/mbeng/OneDrive/Desktop/mass_buildings'
def locate(pattern, root_path):
for path, dirs, files in os.walk(os.path.abspath(root_path)):
for filename in fnmatch.filter(files, pattern):
yield os.path.join(path, filename)
path = [f for f in locate('*.tiff', root_path)]
for file in path:
i = Image.open(file)
#fname = file[file.find('mass_buildings\\'):]
fname = file.replace('D:\\Users\\mbeng\\OneDrive\\Desktop\\mass_buildings', 'D:\\Users\\mbeng\\OneDrive\\Desktop\\resized2')
#fname = fname.replace('\\', '_')
fn, fext = os.path.splitext(fname)
#print(file)
img = i.resize((300, 300))
#print(img)
img.save('{}.tiff'.format(fn), 'TIFF')
when I run the above code, I get the error:
D:\Users\mbeng\Python\PyTorch\python.exe D:/Users/mbeng/Python/FeatureExtract/fils_list.py
Traceback (most recent call last):
File "D:/Users/mbeng/Python/FeatureExtract/fils_list.py", line 68, in <module>
img.save('{}.tiff'.format(fn), 'TIFF')
File "D:\Users\mbeng\Python\PyTorch\lib\site-packages\PIL\Image.py", line 2085, in save
fp = builtins.open(filename, "w+b")
FileNotFoundError: [Errno 2] No such file or directory: 'D:\\Users\\mbeng\\OneDrive\\Desktop\\resized2\\test\\map\\22828930_15.tiff'
Process finished with exit code 1
reized2 is the directory i created to save the processed files in. it contains the directories: train, test and valid, each containing two subdirectories: sat and map. The directory of the original (mass_buildings) files have the same structure as resized2.
How can I make it work?
I figured out a couple of things that were making the code not to work.
the replace() method wasn´t working, so I changed the paths to raw strings and replaced the // with \.
I had to delete the pre-created directory for saving the processed files so that they are created at runtime using Pathlib.Path().mkdir().
root_path= 'D:/Users/mbeng/OneDrive/Desktop/mass_buildings'
def locate(pattern, root_path):
for path, dirs, files in os.walk(os.path.abspath(root_path)):
for filename in fnmatch.filter(files, pattern):
yield os.path.join(path, filename)
path = [f for f in locate('*.tiff', root_path)]
for file in path:
fname = file.replace(r'D:\Users\mbeng\OneDrive\Desktop\mass_buildings', r'D:\\Users\\mbeng\\OneDrive\\Desktop\\resized')
fp = os.path.split(fname)[:-1][0]
base = os.path.basename(fname)
Path(fp).mkdir(parents=True, exist_ok=True)
fn, fext = os.path.splitext(base)
i = Image.open(file)
img = i.resize((700, 700), PIL.Image.NEAREST)
img.save(os.path.join(fp, '{}.tiff'.format(fn)))
I'm running the following script to unzip several zipfiles from a directory:
import os
from zipfile import ZipFile
dir_name = f"C:/my_path/zip/{name}"
def main():
for item in os.listdir(dir_name):
with ZipFile(item, 'r') as zipObj:
listOfFileNames = zipObj.namelist()
for fileName in listOfFileNames:
if fileName.endswith('.csv'):
for i in fileName:
zipObj.extract(fileName, f"C:/my_path/csv/{name}")
if __name__ == '__main__':
main()
The point is that I've confirmed a hundred times that the zip files are stored in the right path but I can't run the script but I did succesfully in another computer (?)
What I am doing wrong?
EDIT:
This is the entire error message:
FileNotFoundError Traceback (most recent call last)
<ipython-input-6-7f26de4464fe> in <module>
18
19 if __name__ == '__main__':
---> 20 main()
<ipython-input-6-7f26de4464fe> in main()
10
11 for item in os.listdir(dir_name): # Iterate over the zip file
---> 12 with ZipFile(item, 'r') as zipObj: # Create a ZipFile Object and load sample.zip in it
13 listOfFileNames = zipObj.namelist() # Get a list of all archived file names from the zip
14 for fileName in listOfFileNames: # Iterate over the file names
C:\Anaconda\lib\zipfile.py in __init__(self, file, mode, compression, allowZip64)
1111 while True:
1112 try:
-> 1113 self.fp = io.open(file, filemode)
1114 except OSError:
1115 if filemode in modeDict:
FileNotFoundError: [Errno 2] No such file or directory: 'my_file.zip'
This might be the key:
Revise this line:
dir_name = f"C:/my_path/zip/{name}"
To:
dir_name = "C:/my_path/zip"
Why?
Because os.listdir is looking in a directory named "C:/my_path/zip/{name}" but it doesn't exist as the variable name has not been declared. Thus the error below, as triggered on my machine (albeit Linux, hence the non-Windows type path):
FileNotFoundError: [Errno 2] No such file or directory: '/devmt/services/{name}'
Then, apply this logic throughout the rest of your function and perhaps use os.path.join to stitch your path and filenames together.
import os
filenames = os.listdir(input("Enter the Path to Directory:"))
for filename in filenames:
os.rename(filename, filename.replace(".", " "))
Traceback (most recent call last):
File "d:/programming/Source Code/projectsssssss/p_test/rename.py", line 8, in
os.rename(filename, filename.replace(".", " "))
FileNotFoundError: [WinError 2] The system cannot find the file specified: 'sdcasd.asdwecwe3.acac3.3ca.wca.txt' -> 'sdcasd asdwecwe3 acac3
3ca wca txt'
os.listdir() will only give you a filename, you have to add a path to it, in order to rename it
import os
path = "/www/css/"
filenames = os.listdir(path)
for filename in filenames:
old_file = os.path.join(path, filename)
new_file = os.path.join(path, filename.replace(".", " "))
os.rename(old_file, new_file)
I Was trying to run this piece of code to generate a directory and a file inside it, the thing is that both the of them won't get created until the program is terminated.
What can i do to create them while the program is still running?
def makeDirectory(dirRoot, path, fileName, fromaddr):
try:
os.mkdir(dirRoot, 0o0755)
except OSError as e:
print(e)
try:
os.mkdir(path, 0o0755)
except OSError as e:
print(e)
path = os.path.abspath(path + '/')
completeName = os.path.join(path, 'foo.txt')
cnt = open(completeName, 'a')
cnt.close()
This will not download the contents of sub-directories; how can I do so?
import ftplib
import configparser
import os
directories = []
def add_directory(line):
if line.startswith('d'):
bits = line.split()
dirname = bits[8]
directories.append(dirname)
def makeDir(archiveTo):
for dir in directories:
newDir = os.path.join(archiveTo, dir)
if os.path.isdir(newDir) == True:
print("Directory \"" + dir + "\" already exists!")
else:
os.mkdir(newDir)
def getFiles(archiveTo, ftp):
files = ftp.nlst()
for filename in files:
try:
directories.index(filename)
except:
ftp.retrbinary('RETR %s' % filename, open(os.path.join(archiveTo, filename), 'wb').write)
def runBackups():
#Load INI
filename = 'connections.ini'
config = configparser.SafeConfigParser()
config.read(filename)
connections = config.sections()
i = 0
while i < len(connections):
#Load Settings
uri = config.get(connections[i], "uri")
username = config.get(connections[i], "username")
password = config.get(connections[i], "password")
backupPath = config.get(connections[i], "backuppath")
archiveTo = config.get(connections[i], "archiveto")
#Start Back-ups
ftp = ftplib.FTP(uri)
ftp.login(username, password)
ftp.cwd(backupPath)
#Map Directory Tree
ftp.retrlines('LIST', add_directory)
#Make Directories Locally
makeDir(archiveTo)
#Gather Files
getFiles(archiveTo, ftp)
#End connection and increase counter.
ftp.quit()
i += 1
print()
print("Back-ups complete.")
print()
this should do the trick :)
import sys
import ftplib
import os
from ftplib import FTP
ftp=FTP("ftp address")
ftp.login("user","password")
def downloadFiles(path,destination):
#path & destination are str of the form "/dir/folder/something/"
#path should be the abs path to the root FOLDER of the file tree to download
try:
ftp.cwd(path)
#clone path to destination
os.chdir(destination)
os.mkdir(destination[0:len(destination)-1]+path)
print destination[0:len(destination)-1]+path+" built"
except OSError:
#folder already exists at destination
pass
except ftplib.error_perm:
#invalid entry (ensure input form: "/dir/folder/something/")
print "error: could not change to "+path
sys.exit("ending session")
#list children:
filelist=ftp.nlst()
for file in filelist:
try:
#this will check if file is folder:
ftp.cwd(path+file+"/")
#if so, explore it:
downloadFiles(path+file+"/",destination)
except ftplib.error_perm:
#not a folder with accessible content
#download & return
os.chdir(destination[0:len(destination)-1]+path)
#possibly need a permission exception catch:
with open(os.path.join(destination,file),"wb") as f:
ftp.retrbinary("RETR "+file, f.write)
print file + " downloaded"
return
source="/ftproot/folder_i_want/"
dest="/systemroot/where_i_want_it/"
downloadFiles(source,dest)
This is a very old question, but I had a similar need that i wanted to satisfy in a very general manner. I ended up writing my own solution that works very well for me. I've placed it on Gist here https://gist.github.com/Jwely/ad8eb800bacef9e34dd775f9b3aad987
and pasted it below in case i ever take the gist offline.
Example usage:
import ftplib
ftp = ftplib.FTP(mysite, username, password)
download_ftp_tree(ftp, remote_dir, local_dir)
The code above will look for a directory called "remote_dir" on the ftp host, and then duplicate the directory and its entire contents into the "local_dir".
It invokes the script below.
import ftplib
import os
def _is_ftp_dir(ftp_handle, name, guess_by_extension=True):
""" simply determines if an item listed on the ftp server is a valid directory or not """
# if the name has a "." in the fourth to last position, its probably a file extension
# this is MUCH faster than trying to set every file to a working directory, and will work 99% of time.
if guess_by_extension is True:
if name[-4] == '.':
return False
original_cwd = ftp_handle.pwd() # remember the current working directory
try:
ftp_handle.cwd(name) # try to set directory to new name
ftp_handle.cwd(original_cwd) # set it back to what it was
return True
except:
return False
def _make_parent_dir(fpath):
""" ensures the parent directory of a filepath exists """
dirname = os.path.dirname(fpath)
while not os.path.exists(dirname):
try:
os.mkdir(dirname)
print("created {0}".format(dirname))
except:
_make_parent_dir(dirname)
def _download_ftp_file(ftp_handle, name, dest, overwrite):
""" downloads a single file from an ftp server """
_make_parent_dir(dest)
if not os.path.exists(dest) or overwrite is True:
with open(dest, 'wb') as f:
ftp_handle.retrbinary("RETR {0}".format(name), f.write)
print("downloaded: {0}".format(dest))
else:
print("already exists: {0}".format(dest))
def _mirror_ftp_dir(ftp_handle, name, overwrite, guess_by_extension):
""" replicates a directory on an ftp server recursively """
for item in ftp_handle.nlst(name):
if _is_ftp_dir(ftp_handle, item):
_mirror_ftp_dir(ftp_handle, item, overwrite, guess_by_extension)
else:
_download_ftp_file(ftp_handle, item, item, overwrite)
def download_ftp_tree(ftp_handle, path, destination, overwrite=False, guess_by_extension=True):
"""
Downloads an entire directory tree from an ftp server to the local destination
:param ftp_handle: an authenticated ftplib.FTP instance
:param path: the folder on the ftp server to download
:param destination: the local directory to store the copied folder
:param overwrite: set to True to force re-download of all files, even if they appear to exist already
:param guess_by_extension: It takes a while to explicitly check if every item is a directory or a file.
if this flag is set to True, it will assume any file ending with a three character extension ".???" is
a file and not a directory. Set to False if some folders may have a "." in their names -4th position.
"""
os.chdir(destination)
_mirror_ftp_dir(ftp_handle, path, overwrite, guess_by_extension)
this is an alternative. you can try using ftputil package. You can then use it to walk the remote directories and get your files
Using ftp.mlsd() instead of ftp.nlst():
import sys
import ftplib
import os
from ftplib import FTP
def fetchFiles(ftp, path, destination, overwrite=True):
'''Fetch a whole folder from ftp. \n
Parameters
----------
ftp : ftplib.FTP object
path : string ('/dir/folder/')
destination : string ('D:/dir/folder/') folder where the files will be saved
overwrite : bool - Overwrite file if already exists.
'''
try:
ftp.cwd(path)
os.mkdir(destination[:-1] + path)
print('New folder made: ' + destination[:-1] + path)
except OSError:
# folder already exists at the destination
pass
except ftplib.error_perm:
# invalid entry (ensure input form: "/dir/folder/")
print("error: could not change to " + path)
sys.exit("ending session")
# list children:
filelist = [i for i in ftp.mlsd()]
print('Current folder: ' + filelist.pop(0)[0])
for file in filelist:
if file[1]['type'] == 'file':
fullpath = os.path.join(destination[:-1] + path, file[0])
if (not overwrite and os.path.isfile(fullpath)):
continue
else:
with open(fullpath, 'wb') as f:
ftp.retrbinary('RETR ' + file[0], f.write)
print(file[0] + ' downloaded')
elif file[1]['type'] == 'dir':
fetchFiles(ftp, path + file[0] + '/', destination, overwrite)
else:
print('Unknown type: ' + file[1]['type'])
if __name__ == "__main__":
ftp = FTP('ftp address')
ftp.login('user', 'password')
source = r'/Folder/'
dest = r'D:/Data/'
fetchFiles(ftp, source, dest, overwrite=True)
ftp.quit()
Using ftputil, a fast solution could be:
def download(folder):
for item in ftp.walk(folder):
print("Creating dir " + item[0])
os.mkdir(item[0])
for subdir in item[1]:
print("Subdirs " + subdir)
for file in item[2]:
print(r"Copying File {0} \ {1}".format(item[0], file))
ftp.download(ftp.path.join(item[0],file), os.path.join(item[0],file))
It is non-trivial at least. In the simplest case, you only assume you have files and directories. This isn't always the case, there are softlinks and hardlinks and Windows-style shortcut. Softlink and directory shortcut are particularly problematic since they make recursive directory possible, which would confuse naive-ly implemented ftp grabber.
How would you handle such recursive directory depends on your need; you might simply not follow softlinks or you might try to detect recursive links. Detecting recursive link is inherently tricky, you cannot do it reliably.