So, the goal is to create a webpage to load a .shp file into and get a summary of some calculations as a JsonResponse. I have prepared the calculations and everything and it works nicely when I add a manual path to the file in question. However, the goal is for someone else to be able to upload the data and get back the response so I can't hardcode my path.
The overall approach:
Read in a through forms.FileField() and request.FILES['file_name']. After this, I need to transfer this request.FILES object to DataSource in order to read it in. I would rather not upload the file on pc if possible but work directly from the memory.
forms.py
from django import forms
from django.core.files.storage import FileSystemStorage
class UploadFileForm(forms.Form):
# title = forms.CharField(max_length=50)
file = forms.FileField()
views.py
import json
import os
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.template import loader
from django.contrib import messages
from django.views.generic import TemplateView
from django.http import JsonResponse
from django.conf import settings
from .forms import UploadFileForm
from . import models
from django.shortcuts import redirect
from gisapp.functions.functions import handle_uploaded_file, handle_uploaded_file_two
from django.contrib.gis.gdal import DataSource
from django.core.files.uploadedfile import UploadedFile, TemporaryUploadedFile
import geopandas as gpd
import fiona
def upload_file(request):
if request.method == 'POST':
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
f = request.FILES['file']
# geo2 = gpd.read_file(f)
# print(geo2)
# f_path = os.path.abspath(os.path.join(os.path.dirname(f), f))
# f_path = TemporaryUploadedFile.temporary_file_path(UploadedFile(f))
# print(f_path)
# f_path = f.temporary_file_path()
# new_path = request.FILES['file'].temporary_file_path
# print(f'This is file path: {f_path}')
# print(f'This is file path: {new_path}')
# data = DataSource(f'gisapp/data/{f}') -- given an absolute path it works great
data = DataSource(f) -- constantly failing
# data = DataSource(new_path)
# print(f'This is file path: {f_path}')
layer = data[0]
if layer.geom_type.name == "Polygon" or layer.geom_type.name == "LineString":
handle_uploaded_file(request.FILES['file'])
elif layer.geom_type.name == "Point":
handle_uploaded_file_two(request.FILES['file'])
return JsonResponse({"Count": f"{handle_uploaded_file_two(request.FILES['file'])[0]}", "Bounding Box": f"{handle_uploaded_file_two(request.FILES['file'])[1]}"})
# return JsonResponse({"Count": f"{handle_uploaded_file(request.FILES['file'])[0]}", "Minimum": f"{handle_uploaded_file(request.FILES['file'])[1]}", "Maximum": f"{handle_uploaded_file(request.FILES['file'])[1]}"})
# instance = models.GeometryUpload(file_field=request.FILES['file'])
# instance.save()
# # return HttpResponseRedirect('/success/')
else:
form = UploadFileForm()
return render(request, 'upload.html', {'form': form})
Error I get:
django.contrib.gis.gdal.error.GDALException: Invalid data source input type: <class 'django.core.files.uploadedfile.InMemoryUploadedFile'>
Now as you can see from the upload_file() in views.py, I tried a multitude of operations and when I add an absolute path, it works, but besides that I can't seem to upload the file to DataSource so that I can use it in my later analysis.
Looking at how Django handles this, it doesn't appear possible to work off of an in memory file. The path to the file is passed to the C API for OGR which then handles opening the file and reading it in.
A possible solution that I am trying myself is to have the user zip their shape files (.shp,.shx.,dbf etc.) beforehand. The zip file is then uploaded and unzipped. The shp files can then be read. Hope this helps
I face the same problem and my workaround was to save the file upload by the user in a temporary folder, then pass the absolute path of the temporary file to my DataSource. After finish all my process with the temporary file, I deleted.
The downside of this method is the execution time, is slow.
Related
Goal of Code:
User uploads a video to our django site.
Use opencv to split it up into individual frames (this works well).
We store the frames in a unique folder on our backend (this is where the problem lies).
What we need help with:
Creation of a unique folder based on user inputted video with django with correct path.
Save the frames of the video into that folder.
The code thus far:
from django.db import models
from django.urls import reverse
import uuid
import cv2
import os
from django.db import models
from PIL import Image, ImageFilter
from django.urls import reverse
from PIL import Image
class Video(models.Model):
vid = models.FileField(upload_to=image_upload_location(filename='jpg'))
img = models.ImageField(upload_to=image_upload_location(filename='jpg'))
date_added = models.DateTimeField(auto_now_add=True)
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
def mp4_to_image(self, *args, **kwargs):
super().save(*args, **kwargs)
This is part we struggling with, creating a unique folder for each video input.
This code works on my local computer path however we need to integrate it with django framework
try:
if not os.exists('exit_folder_path'):
os.makedirs('exit_folder_path')
except OSError:
print ('Error: Creating directory')
Here we use opencv to split the video into individual frames and then save them.
The imwrite() function saves the frame as a jpeg in path defined by name parameter.
We would like these files to be saved to the folder path defined in previous section.
cap = cv2.VideoCapture(self.vid.path)
currentFrame = 0
while (True):
# ret returns a Boolean Value if the frame can be read
ret, frame = cap.read()
# If the frame can't be read then we exit the loop
if ret == False:
break
# Saves image of the current frame in jpg file
name = 'exit_folder_path' + str(currentFrame).zfill(6) + '.jpg'
cv2.imwrite(name, frame)
currentFrame += 1
cap.release()
Problems:
The code saves the user inputted video to a folder called jpg, but no folder is created with the individual frames from the video.
Any and all help is much appreciated.
I guess you need a media folder.
Check here -> https://docs.djangoproject.com/en/3.1/topics/files/
I am trying to make this web app to work but I am getting an error. these are the steps that web app is supposed to handle:
import a file
run the python script
export the results
when I run python script independently( without interfering with flask), it works fine( I use Jupyter notebook) on the other hand, when I run it with flask (from prompt) I get an error:
File "app.py", line 88, in <module>
for name, df in transformed_dict.items():
NameError: name 'transformed_dict' is not defined
Any idea of how can I make this web app to work?
This is my first time using flask and I will appreciate any suggestions or guidance.
python file & html file
from flask import Flask,render_template,request,send_file
from flask_sqlalchemy import SQLAlchemy
import os
import pandas as pd
from openpyxl import load_workbook
import sqlalchemy as db
def transform(df):
# Some data processing here
return df
app=Flask(__name__)
#app.route('/')
def index():
return render_template('firstpage.html')
#app.route('/upload',methods=['Get','POST'])
def upload():
file=request.files['inputfile']
xls=pd.ExcelFile(file)
name_dict = {}
snames = xls.sheet_names
for sn in snames:
name_dict[sn] = xls.parse(sn)
for key, value in name_dict.items():
transform(value)
transformed_dict={}
for key, value in name_dict.items():
transformed_dict[key]=transform(value)
#### wirte to excel example:
writer = pd.ExcelWriter("MyData.xlsx", engine='xlsxwriter')
for name, df in transformed_dict.items():
df.to_excel(writer, sheet_name=name)
writer.save()
if __name__=='__main__':
app.run(port=5000)
Your block:
#### wirte to excel example:
writer = pd.ExcelWriter("MyData.xlsx", engine='xlsxwriter')
for name, df in transformed_dict.items():
df.to_excel(writer, sheet_name=name)
writer.save()
should be part of your upload() function since that's where you define and fill transformed_dict. You just need to match the indentation there to the block above it.
The current error is coming up because it's trying to run that code as soon as you start your script, and transformed_dict doesn't exist at that point.
I want to create a web service using Python.
The service contents are as follows.
Receive music files from users
Analyze the received file and create a new music file. (The function implementation has already been completed using the python library.)
Output the newly created music file to the user.
The main goal is to analyze the music file and create a new one, and input and output the music file for this purpose is a secondary function. So I have no knowledge of creating web services and need help to implement this feature. I went to the internet and implemented it to some extent, but I get an error that I don't know why.
I Upload the code for app.py and __init__.py. I also uploaded the structure of the directory. It looks like something needs to be fixed to use the functions inside the controller. I've been searching the internet and YouTube for more than twelve hours, but I don't really know. I created a class named NoteConvertor and MarcovMatrix in the controller directory.
It runs until I go to http://127.0.0.1:5000/ and receive a file. Then I get the following error:
File "C:\Users\K\git\flask\gp\app.py", line 24, in wav_transform
sr, data = scipy.io.wavfile('static/file1.wav')
TypeError: 'module' object is not callable
-gp
|----controller
| MarcovMatrix.py
| MatrixBuilder.py
| NoteConvertor.py
|----static
| style.css
|----templates
| index.html
| upload.html
| wavplay.html
|----__init__.py
|----app.py
app.py
import flask
from flask import Flask, request, render_template
from werkzeug.utils import secure_filename
import scipy
import numpy as np
from scipy import misc
import pysynth as ps
from scipy.io import wavfile
app = Flask(__name__)
#app.route('/')
#app.route("/upload")
def index():
return render_template('index.html')
#app.route('/uploader', methods=['GET', 'POST'])
def wav_transform():
if request.method == 'POST':
f = request.files['file']
f.save(f'static/file1.wav')
sr, data = scipy.io.wavfile('static/file1.wav')
empty_notes = controller.NoteConvertor(data)
notes = empty_notes.convertor()
song = sum(notes, [])
matrix = controller.MarcovMatrix(song)
start_note = ['e4', 4]
random_song = []
for i in range(0, 100):
start_note = matrix.next_note(start_note)
random_song.append(start_note)
ps.make_wav(random_song, fn='static/random.wav')
return render_template('wavplay.html')
if __name__ == '__main__':
app.run(debug = True)
__init__.py
from gp.controller import MarcovMatrix
from gp.controller import MatrixBuilder
from gp.controller import NoteConvertor
I would like to tell you how to use the functions written in python (inside the controller) inside the flask.
I would be very grateful if you would like to know how to do what I want to implement. Please understand this question because learning the basics of the web is not my main goal.
This is an import issue. You have the choice of either
import scipy
...
sr, data = scipy.io.wavfile('static/file1.wav')
or
from scipy.io import wavfile
...
sr, data = wavfile('static/file1.wav')
I have created a flask application, where I am uploading a file and then predicting the type of the file. I want to write unit test case for the same. I am new to unit test in python and therefore very confused!. There are 2 parts to my code, the first is the Main function, which then calls the classification method.
main.py - here the file is being uploaded and then we call the func_predict function which returns the output
upload_parser = api.parser()
upload_parser.add_argument('file', location='files',
type=FileStorage, required=True)
#api.route('/classification')
#api.expect(upload_parser)
class classification(Resource):
def post(self):
"""
predict the document
"""
args = upload_parser.parse_args()
uploaded_file = args['file']
filename = uploaded_file.filename
prediction,confidence = func_predict(uploaded_file)
return {'file_name':filename,'prediction': prediction,'confidence':confidence}, 201
predict.py : this file contains the func_predict function which does the actual prediction work. It takes the uploaded file as an input
def func_predict(file):
filename = file.filename #filename
extension = os.path.splitext(filename)[1][1:].lower() #file_extension
path = os.path.join(UPLOAD_FOLDER, filename) #store the temporary path of the file
output = {}
try:
# Does some processing.... some lines which are not relevant and then returns the two values
return (''.join(y_pred),max_prob)
Now my confusion is, How do i mock the uploaded file, the uploaded file is of FileStorage type. Also which method should i perform the testing for, should it be the '/classification' or the func_predict.
I have tried the below method, though I did not get any success in this.
I created a test.py file and imported the classification method from main.py and then passed a filename to the data
from flask import Flask, Request
import io
import unittest
from main import classification
class TestFileFail(unittest.TestCase):
def test_1(self):
app = Flask(__name__)
app.debug = True
app.request_class = MyRequest
client = app.test_client()
resp = client.post(
'/classification',
data = {
'file': 'C:\\Users\\aswathi.nambiar\\Desktop\\Desktop docs\\W8_ECI_1.pdf'
}, content_type='multipart/form-data'
)
print(resp.data)
self.assertEqual(
'ok',
resp.data,
)
if __name__ == '__main__':
unittest.main()
I am completely lost! I know there have been earlier questions, but I am not able to figure out .
I have finally stumbled upon how to test it, in case anybody was looking out for something similar.
from predict_main_restplus import func_predict
from werkzeug.datastructures import FileStorage
file = None
def test_classification_correct():
with open('W8-EXP_1.pdf', 'rb') as fp:
file = FileStorage(fp)
a , b = func_predict(file)
assert (a, b) == ('W-8EXP',90.15652760121652)
So, here we are testing the prediction function in predict.py, it returns two values, prediction result and the confidence of the prediction. We can mock the upload using open(file) and then wrapping it with FileStorage. This worked for me.
I've written a very tiny script in python scrapy to parse name, street and phone number displayed across multiple pages from yellowpage website. When I run my script i find it working smoothly. However, the only problem i encounter is the way data are getting scraped in csv output. It is always a line (row) gap between two rows. What I meant is: data are getting printed in every other row. Seeing the picture below you will get to know what I meant. If it were not for scrapy, I could have used [newline='']. But, unfortunately I am totally helpless here. How can i get rid of blank lines coming along in the csv output? Thanks in advance to take a look into it.
items.py includes:
import scrapy
class YellowpageItem(scrapy.Item):
name = scrapy.Field()
street = scrapy.Field()
phone = scrapy.Field()
Here is the spider:
import scrapy
class YellowpageSpider(scrapy.Spider):
name = "YellowpageSp"
start_urls = ["https://www.yellowpages.com/search?search_terms=Pizza&geo_location_terms=Los%20Angeles%2C%20CA&page={0}".format(page) for page in range(2,6)]
def parse(self, response):
for titles in response.css('div.info'):
name = titles.css('a.business-name span[itemprop=name]::text').extract_first()
street = titles.css('span.street-address::text').extract_first()
phone = titles.css('div[itemprop=telephone]::text').extract_first()
yield {'name': name, 'street': street, 'phone':phone}
Here is how the csv output looks like:
Btw, the command I'm using to get csv output is:
scrapy crawl YellowpageSp -o items.csv -t csv
You can fix it by creating a new FeedExporter. Change your settings.py as below
FEED_EXPORTERS = {
'csv': 'project.exporters.FixLineCsvItemExporter',
}
create a exporters.py in your project
exporters.py
import io
import os
import six
import csv
from scrapy.contrib.exporter import CsvItemExporter
from scrapy.extensions.feedexport import IFeedStorage
from w3lib.url import file_uri_to_path
from zope.interface import implementer
#implementer(IFeedStorage)
class FixedFileFeedStorage(object):
def __init__(self, uri):
self.path = file_uri_to_path(uri)
def open(self, spider):
dirname = os.path.dirname(self.path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
return open(self.path, 'ab')
def store(self, file):
file.close()
class FixLineCsvItemExporter(CsvItemExporter):
def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
super(FixLineCsvItemExporter, self).__init__(file, include_headers_line, join_multivalued, **kwargs)
self._configure(kwargs, dont_fail=True)
self.stream.close()
storage = FixedFileFeedStorage(file.name)
file = storage.open(file.name)
self.stream = io.TextIOWrapper(
file,
line_buffering=False,
write_through=True,
encoding=self.encoding,
newline="",
) if six.PY3 else file
self.csv_writer = csv.writer(self.stream, **kwargs)
I am on Mac, so can't test its windows behavior. But if above doesn't work then change below part of code and set newline="\n"
self.stream = io.TextIOWrapper(
file,
line_buffering=False,
write_through=True,
encoding=self.encoding,
newline="\n",
) if six.PY3 else file