I have the following code. It works to serve the index.html or any file located in the home/ directory, but will not serve any files that are located in nested directories, such as home/image/xyz.png.
import os
import sys
import ctypes
import pprint
import bottle
import socket
from bottle import *
#global functions
def ReadStringFromFile( file ):
f = open(file, 'r')
data = f.read()
f.close()
return data
def getBits():
#get the system bits
sysBit = (ctypes.sizeof(ctypes.c_voidp) * 8)
if((not(sysBit == 32)) and (not (sysBit == 64))):
sysBit = 0
return sysBit
class DevelServer( object ):
addr = ''
def __init__( self ):
self.addr = socket.gethostbyname( socket.gethostname() )
pass
def index( self ):
return ReadStringFromFile('home/index.html')
#these are not the URLs you are looking for
def error404( self, error):
return '<br><br><br><center><h1>Go home, You\'re drunk.'
def send_static( self, filename):
print (static_file(filename, root=os.path.join(os.getcwd(), 'home')))
return static_file(filename, root=os.path.join(os.getcwd(), 'home'))
#start hosting the application
def startThis( self ):
bottle.run(host=self.addr, port=80, debug=True)
#instatiate the main application class an initalize all of the app routes
def Main():
ThisApp = DevelServer()
bottle.route('/')(ThisApp.index)
bottle.route('/home/<filename>')(ThisApp.send_static)
bottle.error(404)(ThisApp.error404)
ThisApp.startThis()
Main()
Change this line:
bottle.route('/home/<filename>')(ThisApp.send_static)
to this:
bottle.route('/home/<filename:path>')(ThisApp.send_static)
(Add ":path".)
The Bottle docs explain why:
The static_file() function is a helper to serve files in a safe and
convenient way (see Static Files). This example is limited to files
directly within the /path/to/your/static/files directory because the
wildcard won’t match a path with a slash in it. To serve
files in subdirectories, change the wildcard to use the path filter...
I ran your code with my modification and it worked for me--does it work for you, too?
Related
I have a task to make an iteration from "aaa, aab, aac...aa1...999" using multiprocessing. Though it works, if I make this more than 5 symbols long - it will crush. Please help to make it lighter!
This iteration eventually gets written into a txt file
import logging
import multiprocessing
import pathlib
import uuid
from itertools import product
from logging import Logger
from multiprocessing import Pool
from string import ascii_lowercase, digits
from typing import Iterable
STORAGE = pathlib.Path(__file__).parent.joinpath("words_storage")
def init_logger() -> Logger:
logger = logging.getLogger()
logging.basicConfig(level=logging.INFO)
return logger
class Combinator:
def __init__(
self,
alphabet: Iterable,
str_len: int,
purge_after: bool = True,
logger: Logger = init_logger(),
):
self.alphabet = alphabet
self.str_len = abs(str_len)
self.purge_after = purge_after
self.logger = logger
def product(self):
self.logger.info("[START] Combining started [START]")
self.pre_clean()
try:
with Pool(processes=multiprocessing.cpu_count() - 1) as pool:
result = pool.map(self._get_for_char, self.alphabet)
self.logger.info("[PROCESS] List of filenames received [PROCESS]")
for name in result:
self.logger.info(
f"[PROCESS] Extracting combinations from {name} [PROCESS]"
)
with open(
STORAGE.joinpath("result/result.txt"), mode="a"
) as file_result:
with open(name) as producer:
file_result.write(producer.read())
finally:
if self.purge_after:
self.logger.info(
"[CLEANING] Purging created .txt files in storage [CLEANING]"
)
Combinator.purge_files()
self.logger.info(
"[END] All combinations created successfully. See result.txt file [END]"
)
def combinations_generator(self, char):
for starter_combination in product(self.alphabet, repeat=self.str_len - 1):
combination = [char]
combination.extend(starter_combination)
yield combination
def _get_for_char(self, char) -> pathlib.Path:
self.logger.info(f"[PROCESS] Processing char: {char} [PROCESS]")
file_name = STORAGE.joinpath(f"{uuid.uuid4()}.txt")
result = self.combinations_generator(char)
with open(file_name, mode="w") as file:
file.write(
Combinator.writer("".join(combination) for combination in result)
)
return file_name
def pre_clean(self):
try:
STORAGE.joinpath("result/result.txt").unlink()
self.logger.info("[PRE-CLEAN] Result directory cleared [PRE-CLEAN]")
except FileNotFoundError:
self.logger.info("[PRE-CLEAN] Result directory already clear [PRE-CLEAN]")
#staticmethod
def writer(result_list: Iterable[str]) -> str:
return "\n".join(result_list)
#staticmethod
def purge_files():
for file in STORAGE.glob("*.txt"):
STORAGE.joinpath(file).unlink()
if __name__ == "__main__":
repeats = 5
alphabet_for_combinations = ascii_lowercase + digits
combinator = Combinator(alphabet_for_combinations, repeats, purge_after=True)
combinator.product()
I tried regulating the number of processes and and the maxtasksperchild number, but it didn't help.
Their is a folder in my PC with Linux OS, which contains a website (webpages etc.). The webpages and other complimentary files in the folder use cdns to bring resources like jquery, datatables etc.
I want to make these resources offline. I know I can manually search all files for occurrence of "http" keyword, download files from these URLs keep them in folder and accordingly change source file path. But as these are too many files it seems troublesome. I want to ask is there any better and elegant way of doing so. Thanks in advance
I made a python script to do the job:
import re
import os
import aiohttp
import asyncio
import pathlib
import string
import random
import chardet
# Decode byte sequence using chardet to avoid "Type error"
def decode_bytes(byte_sequence):
result = chardet.detect(byte_sequence)
encoding = result['encoding']
return byte_sequence.decode(encoding)
VALID_URL_REGEX = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_#.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
# Downloader, I lazily have used resp.status as success criteria but it have logical issue you can also include other logic
async def download_file(session, url, local_path):
async with session.get(url, allow_redirects=True, ssl=False) as resp:
if resp.status == 200:
print("Content path is "+str(local_path))
with open(local_path, "wb") as f:
while True:
print(local_path)
chunk = await resp.content.read(4196)
if not chunk:
break
chunk = chunk.encode("utf-8")
f.write(chunk)
downloaded_urls = set()
async def process_file(file_path, session):
print("File during Read "+str(file_path))
with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
contents = f.read()
try:
contents = decode_bytes(contents)
except UnicodeDecodeError as e:
# To avoid Type error
print(f"Error decoding file {file_path}: {e}")
return
urls = re.findall(VALID_URL_REGEX, contents)
try:
for url in urls:
file_name = url.split("/")[-1]
if len(file_name)==0:
continue
if url in downloaded_urls:
local_path = downloaded_urls[url]
# generating random strings to avoid same file name but different urls
res = ''.join(random.choices(string.ascii_uppercase +string.digits, k=5))
file_name=res+file_name
local_path = os.path.join("downloaded", file_name)
if not os.path.exists(local_path):
await download_file(session, url, local_path)
# To avoid redownloading
downloaded_urls.add(url)
contents = contents.replace(url, local_path)
except:
pass
print("File during write "+str(file_path))
with open(file_path, "w", encoding="utf-8", errors="ignore") as f:
f.write(contents)
async def process_directory(directory):
if not os.path.exists("downloaded"):
os.makedirs("downloaded")
conn = aiohttp.TCPConnector(limit=2200,limit_per_host=20,ttl_dns_cache=22)
async with aiohttp.ClientSession(connector=conn) as session:
tasks = []
try:
for filepath in pathlib.Path(directory).glob('**/*'):
fp=filepath.absolute()
if str(fp).endswith(".md") or str(fp).endswith(".txt"):
continue
if os.path.isfile(fp):
tasks.append(process_file(fp, session))
except:
pass
await asyncio.gather(*tasks)
if __name__ == '__main__':
directory = input("Enter root directory") asyncio.run(process_directory(directory))
I will also try "substitution" module and update answer accordingly.
I’m writing a small pywin32 program that converts text files to docx/pdf and that’s all chill and working. BUT… I haven’t figured out a way to unittest the coverter that would not require creating tempfiles. The proccess is really slow.
Is there a commonly accepted good practice on unittesting operations on files?
The converter + unittests below.
# file converter.py
import win32com.client as win32
class Converter:
def __init__(self):
self.word = win32.Dispatch('Word.Application')
self.word.Visible = False
self._word_format = {
'docx': 12,
'pdf': 17
}
#property
def word_format(self):
return self._word_format
def open_and_close(func):
def wrapper(self, fpath):
doc = self.word.Documents.Open(str(fpath))
doc.Activate()
func(self, fpath)
doc.Close()
return wrapper
#open_and_close
def to_docx(self, fpath):
new_fpath = str(fpath).replace('.doc', '.docx')
self.word.ActiveDocument.SaveAs(new_fpath, FileFormat=self.word_format['docx'])
#open_and_close
def to_pdf(self, fpath):
new_fpath = str(fpath).replace('.doc', '.pdf')
self.word.ActiveDocument.SaveAs(new_fpath, FileFormat=self.word_format['pdf'])
def __del__(self):
self.word.Quit()
# file test_converter.py
import unittest
import tempfile
import os
from pathlib import Path
from converter import Converter
class TestConverterToDocx(unittest.TestCase):
#classmethod
def setUpClass(cls):
cls.converter = Converter()
def setUp(self):
self.tempdir = tempfile.TemporaryDirectory()
for _ in range(5):
tempfile.NamedTemporaryFile(dir=self.tempdir.name,
suffix='.doc', delete=False)
def tearDown(self):
self.tempdir.cleanup()
def test_to_docx(self):
for root, dirs, files in os.walk(self.tempdir.name):
for file in files:
self.converter.to_docx(str(Path(root) / file))
files = [file for file in os.listdir(self.tempdir.name)
if file.endswith('.docx')]
self.assertEqual(len(files), 5)
def test_to_pdf(self):
for root, dirs, files in os.walk(self.tempdir.name):
for file in files:
self.converter.to_pdf(str(Path(root) / file))
files = [file for file in os.listdir(self.tempdir.name)
if file.endswith('.pdf')]
self.assertEqual(len(files), 5)
I have previously used importlib as a dynamic importing from a class by doing this:
def load_store(store: str) -> importlib:
"""
Imports the correct path for given store
:param store:
:return:
"""
mod = importlib.import_module(f"lib.vendors.{store}")
class_pointer = getattr(mod, store)()
return class_pointer
However the problem I have seen is that for some reason it calls the importlib 602 times!! whenever I do have this function
on a code that only calls the function once.
from lib.scraper import product_data
from lib.utils import load_store
# To test specific store and link
store: str = "footlockerse"
link: str = "https://www.footlocker.se/en/product/BarelyGreen-Black-White/316700362904"
# -------------------------------------------------------------------------
# Utils
# -------------------------------------------------------------------------
store_class = load_store(store=store) # <--- Calls it only once
def main():
product_data(store_class=store_class)
store = store, link = link, params = "product_page")
if __name__ == '__main__':
main()
I have later on tested to call the import static and the issue went away, However my problem is that I do have around 46 imports that I need to implement and I wonder if I could somehow import only the needed import by given the "store" variable, for example if I have given footlockerse then we import only footlockerse, is that possible?
e.g.
test = "footlockerse"
from lib.vendors.test import test
I've written some python code that needs to read a config file at /etc/myapp/config.conf . I want to write a unit test for what happens if that file isn't there, or contains bad values, the usual stuff. Lets say it looks like this...
""" myapp.py
"""
def readconf()
""" Returns string of values read from file
"""
s = ''
with open('/etc/myapp/config.conf', 'r') as f:
s = f.read()
return s
And then I have other code that parses s for its values.
Can I, through some magic Python functionality, make any calls that readconf makes to open redirect to custom locations that I set as part of my test environment?
Example would be:
main.py
def _open_file(path):
with open(path, 'r') as f:
return f.read()
def foo():
return _open_file("/sys/conf")
test.py
from unittest.mock import patch
from main import foo
def test_when_file_not_found():
with patch('main._open_file') as mopen_file:
# Setup mock to raise the error u want
mopen_file.side_effect = FileNotFoundError()
# Run actual function
result = foo()
# Assert if result is expected
assert result == "Sorry, missing file"
Instead of hard-coding the config file, you can externalize it or parameterize it. There are 2 ways to do it:
Environment variables: Use a $CONFIG environment variable that contains the location of the config file. You can run the test with an environment variable that can be set using os.environ['CONFIG'].
CLI params: Initialize the module with commandline params. For tests, you can set sys.argv and let the config property be set by that.
In order to mock just calls to open in your function, while not replacing the call with a helper function, as in Nf4r's answer, you can use a custom patch context manager:
from contextlib import contextmanager
from types import CodeType
#contextmanager
def patch_call(func, call, replacement):
fn_code = func.__code__
try:
func.__code__ = CodeType(
fn_code.co_argcount,
fn_code.co_kwonlyargcount,
fn_code.co_nlocals,
fn_code.co_stacksize,
fn_code.co_flags,
fn_code.co_code,
fn_code.co_consts,
tuple(
replacement if call == name else name
for name in fn_code.co_names
),
fn_code.co_varnames,
fn_code.co_filename,
fn_code.co_name,
fn_code.co_firstlineno,
fn_code.co_lnotab,
fn_code.co_freevars,
fn_code.co_cellvars,
)
yield
finally:
func.__code__ = fn_code
Now you can patch your function:
def patched_open(*args):
raise FileNotFoundError
with patch_call(readconf, "open", "patched_open"):
...
You can use mock to patch a module's instance of the 'open' built-in to redirect to a custom function.
""" myapp.py
"""
def readconf():
s = ''
with open('./config.conf', 'r') as f:
s = f.read()
return s
""" test_myapp.py
"""
import unittest
from unittest import mock
import myapp
def my_open(path, mode):
return open('asdf', mode)
class TestSystem(unittest.TestCase):
#mock.patch('myapp.open', my_open)
def test_config_not_found(self):
try:
result = myapp.readconf()
assert(False)
except FileNotFoundError as e:
assert(True)
if __name__ == '__main__':
unittest.main()
You could also do it with a lambda like this, if you wanted to avoid declaring another function.
#mock.patch('myapp.open', lambda path, mode: open('asdf', mode))
def test_config_not_found(self):
...