I'm trying to automate a daily ftp transfer using a Python3 script. I'm having a small issue though with writing the files were i want them to be. This is what I'm doing:
import time, os
from ftplib import FTP
from datetime import datetime
today=time.strftime('%d%m%y')
dirName='mydir'+today
if not os.path.exists(dirName):
os.mkdir(dirName)
print("Directory " , dirName , " Created ")
else:
print("Directory " , dirName , " already exists")
os.chdir(dirName)
start = datetime.now()
ftp = FTP('ftp')
ftp.login('user','pass')
ftpdir='localdir'+today
ftp.cwd(ftpdir)
# Get All Files
files = ftp.nlst()
# Print out the files
for file in files:
print("Downloading..." + file)
ftp.retrbinary("RETR " + file, open(dirName + file, 'wb').write)
ftp.close()
what I get with this code is that all the downloaded ftp files stay in the folder level above "today" while their filename start with the "today" str.
Can someone give a hand here please.
Thanks in advance
You have to separate the path components. For platform independent solution, use os.path.join:
import os
dirName = os.path.join('mydir', today)
Solved the issue with a bar:
# Print out the files
for file in files:
print("Downloading..." + file)
ftp.retrbinary("RETR " + file, open(dirName + '\\' + file, 'wb').write)
ftp.close()
Related
I am trying to test a python code which moves file from source path to target path . The test is done using pytest in Python3 . But I am facing a roadblock here. It is that , I am trying to remove the source and target paths at end of code completion. For this I am using a command like shutil.rmtree(path) or os.rmdir(path) . This is causing me the error - " [WinError 32] The process cannot access the file because it is being used by another process". Please help me on this. Below is the python pytest code :
import pytest
import os
import shutil
import tempfile
from sample_test_module import TestCondition
object_test_condition = TestCondition()
#pytest.mark.parametrize("test_value",['0'])
def test_condition_pass(test_value):
temp_dir = tempfile.mkdtemp()
temp_src_folder = 'ABC_File'
temp_src_dir = os.path.join(temp_dir , temp_src_folder)
temp_file_name = 'Sample_Test.txt'
temp_file_path = os.path.join(temp_src_dir , temp_file_name)
os.chdir(temp_dir)
os.mkdir(temp_src_folder)
try:
with open(temp_file_path , "w") as tmp:
tmp.write("Hello-World\n")
tmp.write("Hi-All\n")
except IOError:
print("Error has occured , please check it.")
org_val = object_test_condition.sample_test(temp_dir)
print("Temp file path is : " + temp_file_path)
print("Temp Dir is : " + temp_dir)
shutil.rmtree(temp_dir)
print("The respective dir path is now removed.)
assert org_val == test_value
Upon execution of the code , the below error is popping up :
[WinError32] The process cannot access the file because it is being used by another process : 'C:\Users\xyz\AppData\Local\Temp\tmptryggg56'
You are getting this error because the directory you are trying to remove is the current directory of the process. If you save the current directory before calling os.chdir (using os.getcwd()), and chdir back to that directory before removing temp_dir, it should work.
Your code isn't correctly indented, so here is my best guess at what it should look like.
import pytest
import os
import shutil
import tempfile
from sample_test_module import TestCondition
object_test_condition = TestCondition()
#pytest.mark.parametrize("test_value",['0'])
def test_condition_pass(test_value):
temp_dir = tempfile.mkdtemp()
temp_src_folder = 'ABC_File'
temp_src_dir = os.path.join(temp_dir , temp_src_folder)
temp_file_name = 'Sample_Test.txt'
temp_file_path = os.path.join(temp_src_dir , temp_file_name)
prev_dir = os.getcwd()
os.chdir(temp_dir)
os.mkdir(temp_src_folder)
try:
with open(temp_file_path , "w") as tmp:
tmp.write("Hello-World\n")
tmp.write("Hi-All\n")
except IOError:
print("Error has occured , please check it.")
org_val = object_test_condition.sample_test(temp_dir)
print("Temp file path is : " + temp_file_path)
print("Temp Dir is : " + temp_dir)
os.chdir(prev_dir)
shutil.rmtree(temp_dir)
print("The respective dir path is now removed.)
assert org_val == test_value
Can you try to close the temp file before removing
temp.close()
I am trying to donwload a huge zip file (~9Go zipped and ~130GO unzipped) from an FTP with python using the ftplib library but unfortunately when using the retrbinary method, it does create the file in my local diretory but it is not writing into the file. After a while the code runs, I get an timeout error. It used to work fine before, but when I tried to go deeper in the use of sockets by using this code it does not work anymore. Indeed, as the files I am trying to download are huge I want to have more control with the connection to prevent timeout error while downloading the files. I am not very familar with sockets so I may have misused it. I have been searching online but did not find any problems like this. (I tried with smaller files too for test but still have the same issues)
Here are the function that I tried but both have problems (method 1 is not writing to file, method 2 donwloads file but I can't unzip it)
import time
import socket
import ftplib
import threading
# To complete
filename = ''
local_folder = ''
ftp_folder = ''
host = ''
user = ''
mp = ''
# timeout error in method 1
def downloadFile_method_1(filename, local_folder, ftp_folder, host, user, mp):
try:
ftp = ftplib.FTP(host, user, mp, timeout=1600)
ftp.set_debuglevel(2)
except ftplib.error_perm as error:
print(error)
with open(local_folder + '/' + filename, "wb") as f:
ftp.retrbinary("RETR" + ftp_folder + '/' + filename, f.write)
# method 2 works to download zip file, but header error when unziping it
def downloadFile_method_2(filename, local_folder, ftp_folder, host, user, mp):
try:
ftp = ftplib.FTP(host, user, mp, timeout=1600)
ftp.set_debuglevel(2)
sock = ftp.transfercmd('RETR ' + ftp_folder + '/' + filename)
except ftplib.error_perm as error:
print(error)
def background():
f = open(local_folder + '/' + filename, 'wb')
while True:
block = sock.recv(1024*1024)
if not block:
break
f.write(block)
sock.close()
t = threading.Thread(target=background)
t.start()
while t.is_alive():
t.join(60)
ftp.voidcmd('NOOP')
def unzip_file(filename, local_folder):
local_filename = local_folder + '/' + filename
with ZipFile(local_filename, 'r') as zipObj:
zipObj.extractall(local_folder)
And the error I get for method 1:
ftplib.error_temp: 421 Timeout - try typing a little faster next time
And the error I get when I try to unzip after using method 2:
zipfile.BadZipFile: Bad magic number for file header
Alos, regarding this code If anyone could explain what this does concerning socketopt too would be helpful:
ftp.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
ftp.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 75)
ftp.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 60)
Thanks for your help.
i am trying to resolve the path of a file that has been symlink in python on linux os
i have tried this
# sets up the log directory for all files
log_dir = os.path.join(os.path.normpath(os.getcwd() + os.sep + os.pardir),'logs')
try os.readlink. Assuming log_dir is your soft link,
soft_link = os.path.join(
os.path.normpath(os.getcwd() + os.sep + os.pardir),
'logs',
)
abs_path = os.readlink(soft_link)
os.readlink
Return a string representing the path to which the symbolic link points.
I have this part code which i don't understand the for loop part:
also what does file path holds and how it is different than all file path?
all_files_path = glob.glob("ADNI1_Screening_1.5T/ADNI/*/*/*/*/*.nii")
for file_path in all_file_path:
print(os.system("./runROBEX.sh " + file_path + " stripped/" +file_path.split("/")[-1]))
So far I have the following:
source_folder = 'file_location'
for file in os.listdir(source_folder):
if file.startswith('stnet_'):
os.rename(file, file.replace('stnet_a_b', '%s_' % time.ctime(os.path.getctime(file)) + 'stnet_a_b'))
The issue with is is I keep getting FileNotFoundError: [WinError 2] The system cannot find the file specified 'stnet_a_b.raw'
Can someone point out what I'm doing wrong?
Thanks.
os.listdir can only get the filenames without directory, while os.rename, os.path.getctime needs full name with directory(if your current directory is not conincidently file_location then the file will not be found).
You can use os.path.join to get the full name. And if you are on Windows you must make sure filename doesn't contain special characters which your code contains.
dir = r'file_location'
# os.chdir(dir) # in case you don't want to use os.path.join
for filename in os.listdir(dir):
print(filename)
if filename.startswith('stnet_'):
src = os.path.join(dir, filename)
ctime_str = str(time.ctime(os.path.getctime(src)))
ctime_str = ctime_str.replace(':', '').replace(' ', '') # remove special characters
fn_new = filename.replace('stnet_a_b',
'{}_'.format(ctime_str + 'stnet_a_b'))
des = os.path.join(dir, fn_new)
print('src={}, des={}'.format(src, des))
os.rename(src, des)
please try above code.