Just recently I ran 'python manage.py rebuild_index -v2' to build a searchable index for Haystack-Elastisearch project. The results to the rebuild index command were:
Indexing 452 quorum_ sensings
indexed 1 - 452 of 452 (by 8436).
When I check the index there is no data in the index.
What would cause this?
I'm using Django 1.6, Haystack 2.1, and Python 3.3.3
Here is some of my code:
models.py
from django.db import models
class Quorum_Sensing(models.Model):
IUAPAC_NAME = models.CharField(max_length=400)
Inducer_Molecule = models.CharField(max_length=100)
Category_Inducer_Molecule = models.CharField(max_length=40)
Inducer_Name_Synonym_Abbreviation = models.TextField()
Chemical_Nature_Inducer_Molecule = models.CharField(max_length=40)
Natural_Synthetic = models.CharField(max_length=120)
Pubchem_Link_Inducer_Molecules = models.URLField()
Mass_Inducer_Molecule = models.CharField(max_length=50)
Number_Amino_Acids_In_Inducer_Molecule = models.CharField(max_length=60)
IUPAC_Chemical_ID = models.CharField(max_length=60)
QuorumPep_Link = models.URLField()
Amino_Acid_Sequence_Inducer = models.CharField (max_length=300)
SMILES = models.CharField(max_length=250)
Organism_Involved = models.CharField (max_length=200)
Strain1 = models.CharField(max_length=120)
Strain2 = models.CharField(max_length=120)
Strain3 = models.CharField(max_length=120)
Strain4 = models.CharField(max_length=120)
Strain5 = models.CharField(max_length=120)
Taxonomy_Link = models.URLField()
Pathogenic_Or_Not = models.CharField(max_length=120)
Host = models.CharField(max_length=120)
Target_Protein = models.CharField (max_length=160)
Target_Protein_Length = models.CharField (max_length=40)
Gene_Name = models.CharField(max_length=60)
Broad_Functional_Category_Target_Molecule = models.CharField(max_length=300)
GI = models.CharField(max_length=30)
NCBI_Link_Target_Molecule = models.URLField(max_length=400)
Mode_Of_Action = models.TextField()
UNIPROT_Link_Target_Molecule = models.URLField()
EMBL_Link_Target_Molecule = models.URLField()
Related_Citations = models.URLField(max_length=400)
Annotator = models.EmailField()
quorum_sensing_text.txt
{{ object.text }}
{{ object.Category_Inducer_Molecule }}
{{ object.Amino_Acid_Sequence_Inducer }}
{{ object.Organism_Involved }}
{{ object.Target_Protein }}
{{ object.Broad_Functional_Category_Target_Molecule }}
search_indexes.py
from bacterial.models import Quorum_Sensing
from haystack import indexes
class Quorum_SensingIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True, template_name='quorum_sensing_text.txt')
Category_Inducer_Molecule = indexes.CharField(model_attr='Category_Inducer_Molecule', faceted=True)
Amino_Acid_Sequence_Inducer = indexes.CharField(model_attr='Amino_Acid_Sequence_Inducer', faceted=True)
Organism_Involved = indexes.CharField(model_attr='Organism_Involved', faceted=True)
Target_Protein = indexes.CharField(model_attr='Target_Protein', faceted=True)
Broad_Functional_Category_Target_Molecule = indexes.CharField(model_attr='Broad_Functional_Category_Target_Molecule',faceted=True)
def get_model(self):
return Quorum_Sensing
def index_queryset(self, using=None):
return self.get_model().objects.all()
Thank you in advance for any help!
Discovered the issue to this problem. It was a corrupted Haystack fields.py file. Fixed the file and now the "text" field is being populated
Related
Hi Everyone i am working work django framework, where i used to upload excel file in Dailytrip table, current i get car_mumber from car table, but now i need to store car_number from Car_team table also team_id, i am storing car_id and team_id in car_team table also i need to store team_id in dailytrip table automaticlly based on car_id(car_number) i am to much confuse how to i work that, pls help me out
models.py
class Car_team(BaseModel):
team = models.ForeignKey(
Team,
models.CASCADE,
verbose_name='Team',
null=True,
)
car=models.ForeignKey(
Car,
models.CASCADE,
verbose_name='Car',
null=True)
city =models.ForeignKey(
City,
models.CASCADE,
verbose_name='City',
)
start_date=models.DateField(null=True, blank=True)
end_date=models.DateField(null=True, blank=True)
views.py
def add_payout_uber_daily_data(request):
if request.method == 'POST':
form = UberPerformanceDataForm(request.POST, request.FILES, request=request)
if form.is_valid():
date = form.cleaned_data['date']
excel_file = request.FILES['file']
df = pd.read_excel(excel_file)
is_na = pd.isna(df['Date']).sum().sum() + pd.isna(df['Name']).sum().sum() + pd.isna(df['UUID']).sum().sum() + pd.isna(df['Net Fare With Toll']).sum().sum() + pd.isna(df['Trips']).sum().sum() + pd.isna(df['Uber KMs']).sum().sum() + pd.isna(df['CashCollected']).sum().sum() + pd.isna(df['UberToll']).sum().sum() + pd.isna(df['Tips']).sum().sum() + pd.isna(df['Hours Online']).sum().sum() + pd.isna(df['Ratings']).sum().sum() + pd.isna(df['Acceptance Rate']).sum().sum() + pd.isna(df['Cancellation Rate']).sum().sum()
error_list = []
if is_na > 0:
error_list.append('Found #N/A or blank values in the sheet. Please correct and re-upload')
context = {'error_list': error_list, 'menu_payout': 'active','submenu_daily_data': 'active','form': form, }
return render(request, 'add_payout_uber_daily_data.html', context=context)
date_match = True
for d in df['Date']:
if str(d.strftime("%Y-%m-%d")) != str(date):
date_match = False
break
if not date_match:
error_list.append('Some dates are not matching in excel')
if len(error_list) > 0:
context = {'error_list': error_list, 'menu_payout': 'active','submenu_daily_data': 'active','form': form, }
return render(request, 'add_payout_uber_daily_data.html', context=context)
DailyTrip.objects.filter(date=date).update(is_active=0)
for i in df.index:
uuid = df['UUID'][i]
driver_id = None
car_id = None
fleet_id = None
manager_id = None
try:
driver = Driver.objects.get(uber_uuid=uuid)
driver_id = driver.id
except Driver.DoesNotExist:
driver_id = None
#replce car code and store car_number,car_id,team_id via car_team only this logic need to change current get car_number direct car table but we need car_number vai foriegn key
try:
car = Car.objects.get(car_number=df["Car Number"][i])
car_id = car.id
manager_id = car.manager_id
except Car.DoesNotExist:
car_id = None
try:
fleet = Fleet.objects.get(name=df["Fleet Name"][i])
fleet_id = fleet.id
except Fleet.DoesNotExist:
fleet_id = None
name = df['Name'][i]
car_number = df['Car Number'][i]
fare_total = df['Net Fare With Toll'][i]
trips = df['Trips'][i]
pool_trips = 0
hours_online = df['Hours Online'][i]
total_km = df['Uber KMs'][i]
cash_collected = abs(df['CashCollected'][i])
toll = df['UberToll'][i]
tip_amount = df['Tips'][i]
fare_avg = float(fare_total)/int(trips)
fare_per_hour_online = float(fare_total)/float(hours_online)
fare_per_km = fare_total/total_km
trips_per_hour = trips/hours_online
km_per_trip = total_km/trips
rating = df['Ratings'][i]
acceptance_rate_perc = float(df['Acceptance Rate'][i])/100
driver_cancellation_rate = float(df['Cancellation Rate'][i])/100
obj, created = DailyTrip.all_objects.update_or_create(
date=date, uuid=uuid,
defaults={
'car_id': car_id,
'manager_id': manager_id,
'car_number': car_number,
'driver_id': driver_id,
'car_id': car_id,
'fleet_id': fleet_id,
'driver_name': name,
'fare_total': fare_total,
'trips': trips,
'pool_trips': pool_trips,
'hours_online': hours_online,
'total_km': total_km,
'cash_collected': cash_collected,
'toll': toll,
'tip_amount': tip_amount,
'fare_avg': fare_avg,
'fare_per_hour_online':fare_per_hour_online,
'fare_per_km':fare_per_km,
'trips_per_hour': trips_per_hour,
'km_per_trip': km_per_trip,
'rating': rating,
'acceptance_rate_perc': acceptance_rate_perc,
'driver_cancellation_rate': driver_cancellation_rate,
'is_active': 1,
'comments': None}
)
if len(error_list) > 0:
DailyTrip.objects.filter(date=date).update(is_active=0)
context = {'error_list': error_list, 'menu_payout': 'active','submenu_daily_data': 'active','form': form, }
return render(request, 'add_payout_uber_daily_data.html', context=context)
else:
messages.success(request, 'Daily Trips added Successfully...')
return redirect('/fleet/payout/daily_data/add/uber')
else:
form = UberPerformanceDataForm(initial={})
context = {
'menu_payout': 'active',
'submenu_daily_data': 'active',
'form': form,
}
return render(request, 'add_payout_uber_daily_data.html', context=context)
You can try that :
to get car_number from car_team -->
car_team = car_team.objects.objects.all().last() # to get the last car_team for example
car_number = car_team.car.car_number # to get the car number from the car_team
try:
car = Car.objects.get(car_number=df["Car Number"][i])
car_id = car.id
car1=Car_team.objects.filter(car_id=car_id)
if car1:
team_id=car1[0].team_id
else:
team_id=None
except Car.DoesNotExist:
car_id = None
team_id= None
I am having an issue writing a dataframe to my django models.py.
The file is long, but is quite simple in its methodology:
-import modules
-create django database
-requests.get necessary data
-alter data some to fit my goals, save as df
-connect to django db and insert df
My models.py is the following:
from django.db import models
import requests
import pandas as pd
from datetime import timezone
from datetime import datetime
from datetime import date
from datetime import timedelta
import time
from django.conf import settings
from sqlalchemy.engine import create_engine
class cryptoData(models.Model):
coin = models.CharField(max_length=10)
asset_id = models.SmallIntegerField()
time = models.DateTimeField()
close = models.FloatField()
volume = models.BigIntegerField()
market_cap = models.FloatField()
reddit_posts = models.IntegerField()
reddit_comments = models.IntegerField()
tweets = models.IntegerField()
tweet_favorites = models.IntegerField()
social_volume = models.IntegerField()
lunarcrush_key = 'fakekey1234'
def top_coins():
lc_market = requests.get(
url = 'https://api.lunarcrush.com/v2?data=market&',
params = {
'key': lunarcrush_key,
}
)
all_coins = []
for entry in lc_market.json().get('data'):
coin = []
coin.append(entry.get('s'))
coin.append(entry.get('mc'))
all_coins.append(coin)
all_coins.sort(key = lambda x : x[1], reverse = True)
top_ten_coins = all_coins[:10]
return(top_ten_coins)
top_coins_lst = top_coins()
top_coin_names_lst = [x[0] for x in top_coins_lst]
def get_coin_data(key, coin, date_diff, start_date, end_date):
lc = requests.get(
url = 'https://api.lunarcrush.com/v2?data=assets&',
params = {
'key': lunarcrush_key,
'symbol': coin,
'interval': 'day',
'data_points': date_diff,
'start': int(start_date.replace(tzinfo=timezone.utc).timestamp()),
'end': int(end_date.replace(tzinfo=timezone.utc).timestamp())
}
)
metric_names = []
for entry in lc.json().get('data')[0].get('timeSeries'):
for key in entry:
metric_names.append(key) if key not in metric_names else metric_names
metrics_list = []
for entry in lc.json().get('data')[0].get('timeSeries'):
row_list = []
for key in entry:
row_list.append(entry.get(key))
metrics_list.append(row_list)
metrics_df = pd.DataFrame(metrics_list, columns = metric_names)
metrics_df['time'] = metrics_df['time'].apply(lambda x : datetime.utcfromtimestamp(x).strftime('%Y-%m-%d %H:%M:%S'))
metrics_df['coin'] = coin
cols = list(metrics_df)
cols.insert(0, cols.pop(cols.index('coin')))
metrics_df = metrics_df.loc[:, cols]
return(metrics_df)
def get_all_coins_data(coins_list):
appended_data = []
end_date = datetime.now()
start_date = end_date - timedelta(days = 700)
date_diff = (end_date - start_date).days
for coin in coins_list:
appended_data.append(get_coin_data(lunarcrush_key, coin, date_diff, start_date, end_date))
time.sleep(.1)
output = pd.concat(appended_data)
return(output)
df = get_all_coins_data(top_coin_names_lst)
focused_df = df[['coin', 'asset_id', 'time', 'close', 'volume', 'market_cap', 'reddit_posts', 'reddit_comments', 'tweets', 'tweet_favorites', 'social_volume']]
user = settings.DATABASES['default']['USER']
password = settings.DATABASES['default']['PASSWORD']
database_name = settings.DATABASES['default']['NAME']
database_url = 'sqlite://{user}:{password}#localhost:5432/{database_name}'.format(
user=user,
password=password,
database_name=database_name,
)
engine = create_engine(database_url, echo=False)
focused_df.to_sql(cryptoData, con=engine)
When I run the manage.py runserver command, I get the following error:
sqlalchemy.exc.ArgumentError: Invalid SQLite URL: sqlite://user:password#localhost:5432/C:\Users\user\Programming\django_crypto_v6\source\db.sqlite3
Valid SQLite URL forms are:
sqlite:///:memory: (or, sqlite://)
sqlite:///relative/path/to/file.db
sqlite:////absolute/path/to/file.db
I'm struggling to resolve this issue. Any thoughts?
you are using the wrong pattern for SQLite database_url
see the docs at https://docs.sqlalchemy.org/en/14/core/engines.html#sqlite
I am trying to update the record that already exists in the database and therefore I use this code
if 'supplierId' not in req.keys():
return JsonResponse({'code': 0, 'msg': "supplier was not selected", 'result': ''}, safe=False)
assigneeId = User.objects.get(pk=req.get('assigneeId', 1))
responsibleId = User.objects.get(pk=req.get('responsibleId', 1))
redistributionMethod = req.get('redistributionMethod', 0)
amount = req.get('allCost', 0)
procurement_doc = ProcurementDocJournal.objects.get(id=pk)
print(procurement_doc)
procurement_doc.docType = req['docType']
procurement_doc.status = req['status']
procurement_doc.companyId = Company.objects.get(pk=req['companyId'])
procurement_doc.datetime = req['datetime']
procurement_doc.supplierId = Partner.objects.get(pk=req['supplierId'])
procurement_doc.assigneeId = assigneeId
procurement_doc.warehouseId = Warehouse.objects.get(pk=req['warehouseId'])
procurement_doc.responsibleId = responsibleId
procurement_doc.redistributionMethod = redistributionMethod
procurement_doc.amount = amount
procurement_doc.comment = req['comment']
procurement_doc.save(force_update=True, update_fields=['comment', 'amount', 'redistributionMethod',
'responsibleId', 'warehouseId',
'supplierId', 'datetime',
'companyId', 'assigneeId', 'status', 'docType'])
where req contains a request
something like this
{
'docType': 3,
'status': 1,
'companyId': '2',
'warehouseId': '3',
'assigneeId': '5',
'supplierId': '12671',
'responsibleId': '5',
'datetime': '2020-04-01 08:01:00',
'comment': ''
}
As you can see there is a print which assures me that I selected the correct row
when I noticed that these records are not updated I searched for causes and found
this question where the guy who asked says The message field was missing from the model definition
in my case, none of these are missing from the model's description
class ProcurementDocJournal(models.Model):
id = models.IntegerField(primary_key=True, null=False)
docNumber = models.IntegerField()
status = models.IntegerField()
docType = models.IntegerField()
depended = models.IntegerField()
companyId = models.ForeignKey(Company, on_delete=models.CASCADE,
db_column='companyId')
created_at = models.DateTimeField(auto_now_add=True)
datetime = models.DateTimeField()
currencyId = models.ForeignKey(Currency, db_column='currencyId', on_delete=models.CASCADE)
currencyRate = models.FloatField()
redistributionMethod = models.IntegerField()
assigneeId = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ProcurementDocJournal',
db_column='assigneeId')
warehouseId = models.ForeignKey(Warehouse, on_delete=models.CASCADE,
db_column='warehouseId')
responsibleId = models.ForeignKey(User, on_delete=models.CASCADE, related_name='ProcurementDoc',
db_column='responsibleId')
supplierId = models.ForeignKey(Partner, on_delete=models.CASCADE,
db_column='supplierId')
amount = models.FloatField()
comment = models.TextField()
class Meta:
db_table = 'procurementDocJournal'
get_latest_by = 'id'
Edit
I have an action that contains
procurement_doc_journal_item = ProcurementDocJournal.objects.get(id=pk)
currencyId = req['currency']
currency = Currency.objects.get(id=currencyId)
currencyRate = CurrencyRate(date, currency.name)
procurement_doc_journal_item.currencyId = currency
procurement_doc_journal_item.currencyRate = currencyRate['rate']
procurement_doc_journal_item.save()
and works like a charm
there is no any error that logs say
You don't specify if there's any error in your logs. I'd kind of expect to see something since if it's not saving, it must be bombing out before that as no fields are mandatory.
I am, however, not sure you're setting your FKs correctly here:
procurement_doc.companyId = Company(req['companyId'])
Should be
procurement_doc.companyId = Company.objects.get(pk=req['companyId'])
and that assumes companyId is set to a value that exists of course ... Whether this is the source of your issue or not, I'm not sure. I don't see any of your FKs are mandatory.
Edit: I've just tested this on one of my models with a simple id key:
>>> Contact(1)
Contact()
>>> c = Contact(1)
>>> c.name
''
>>> c = Contact.objects.get(pk=1)
>>> c.name
'Mike'
It really doesn't seem to me as though it works ...
Sidenote: You can simplify code like this:
if 'allCost' not in req.keys():
amount = 0
else:
amount = req['allCost']
By doing:
amount = req.get('allCost', 0)
I have a form where I have a text field and a selection field and the question I have is how can I insert the bd in the selection field, this field in the table is an external key and what I need is to insert the value of the selection field.
I would greatly appreciate your help.
#detalle_grupo_estadisticas.route("/show/<id>", methods = ['GET', 'POST'])
def show(id):
title = 'Grupo estadisticas'
page = 1
per_page = 3
CamposGrupo = forms.Fields_Detalle_Grupo_Estadisticas(request.form)
grup_list = GrupoEstadisticas.query.all()
if request.method == 'POST' and CamposGrupo.validate():
grupo_edit = DetalleGrupoEstadisticas.query.filter_by(id_detalle_grupo_estadisticas=id).first()
grupo_edit.nombre = CamposGrupo.Nombre.Data
grupo_edit.id_grupoEstadisticas = CamposGrupo.GrupoEstadistica.Value
db.session.add(grupo_edit)
db.session.commit()
return redirect(url_for('detalle_grupo_estadisticas.index'))
grupo = DetalleGrupoEstadisticas.query.filter_by(id_detalle_grupo_estadisticas=id).first()
CamposGrupo.Nombre.data = grupo.nombre
CamposGrupo.GrupoEstadistica.choices=[(g.id_grupoEstadisticas,g.nombre) for g in grup_list]
return render_template('/detalle_grupo_estadisticas/show.html', title=title, CamposGrupo = CamposGrupo, grupo = grupo)
What I want to insert is the value of the field selection field
these are the models
class DetalleGrupoEstadisticas(db.Model):
__tablename__ = 'detalle_grupo_estadisticas'
id_detalle_grupo_estadisticas = db.Column(db.Integer, primary_key=True)
nombre = db.Column(db.String(80), nullable=False)
id_grupoEstadisticas = db.Column(db.Integer, db.ForeignKey('grupo_estadisticas.id_grupoEstadisticas'))
grupoestadistica = db.relationship('GrupoEstadisticas', back_populates='detallesgrupo', lazy=True)
class GrupoEstadisticas(db.Model):
__tablename__ = 'grupo_estadisticas'
id_grupoEstadisticas = db.Column(db.Integer, primary_key=True)
nombre = db.Column(db.String(80), nullable=False)
descripcion = db.Column(db.String(200), nullable=False)
estadisticas= db.relationship('Estadisticas', back_populates='grupoestadisticas', lazy=True)
detallesgrupo= db.relationship('DetalleGrupoEstadisticas', back_populates='grupoestadistica', lazy=True)
This is how I was able to solve the problem.
view
#detalle_grupo_estadisticas.route("/show/<id>", methods = ['GET', 'POST'])
def show(id):
title = 'Grupo estadisticas'
page = 1
per_page = 3
CamposGrupo = forms.Fields_Detalle_Grupo_Estadisticas(request.form)
grup_list = GrupoEstadisticas.query.all()
if request.method == 'POST' and CamposGrupo.validate():
grupo_edit = DetalleGrupoEstadisticas.query.filter_by(id_detalle_grupo_estadisticas=id).first()
grupo_edit.nombre = CamposGrupo.Nombre.data
grupo_edit.grupoestadistica = CamposGrupo.GrupoEstadistica.data
#this is where I had the error that I was inserting in id_grupoStadisticas and it had to be with the variable of the relationship
db.session.add(grupo_edit)
db.session.commit()
return redirect(url_for('detalle_grupo_estadisticas.index'))
grupo = DetalleGrupoEstadisticas.query.filter_by(id_detalle_grupo_estadisticas=id).first()
CamposGrupo.Nombre.data = grupo.nombre
CamposGrupo.GrupoEstadistica.choices=[(g.id_grupoEstadisticas,g.nombre) for g in grup_list]
return render_template('/detalle_grupo_estadisticas/show.html', title=title, CamposGrupo = CamposGrupo, grupo = grupo)
form
class Fields_Detalle_Grupo_Estadisticas(Form):
Nombre = StringField('Grupo', validators=[InputRequired(), Length(min=2, max=30, message = 'Mayor a 2 caracteres y menor a 30')])
GrupoEstadistica = QuerySelectField('Grupo Estadistica',query_factory=lambda: GrupoEstadisticas.query,
get_pk=lambda g: g.id_grupoEstadisticas,
get_label=lambda g: g.nombre)
Again I have another problem with outputing SQL Alchemy dates in flask/jinja2
my class is defined as
class vw_invoice_header(db.Model):
__tablename__ = "vw_invoice_header"
tax_amount = db.Column(db.Numeric)
total_price = db.Column(db.Numeric)
invc_number = db.Column(db.String(40))
term_code = db.Column(db.String(20))
order_date = db.Column(db.DateTime)
airway_bill = db.Column(db.String(40))
inh_auto_key = db.Column(db.Integer,primary_key=True)
invoice_date = db.Column(db.DateTime)
company_po_number = db.Column(db.String(30))
ship_name = db.Column(db.String(50))
ship_address1 = db.Column(db.String(35))
ship_address2 = db.Column(db.String(35))
ship_address3 = db.Column(db.String(35))
ship_address4 = db.Column(db.String(35))
ship_address5 = db.Column(db.String(35))
attention = db.Column(db.String(50))
phone_number = db.Column(db.String(20))
email_address = db.Column(db.String(60))
bill_name = db.Column(db.String(50))
bill_address1 = db.Column(db.String(35))
bill_address2 = db.Column(db.String(35))
bill_address3 = db.Column(db.String(35))
bill_address4 = db.Column(db.String(35))
bill_address5 = db.Column(db.String(35))
so_numbers = db.Column(db.String(4000))
company_ref_numbers = db.Column(db.String(4000))
currency = db.Column(db.String(40))
is_complete = db.Column(db.String(1))
I'm doing a
invc = vw_invoice_header.query.filter_by(inh_auto_key=20643519).first()
why does
{{invc.invoice_date.strftime('%d %b %y') }}
give me the results expected, yet -
{{ '%d %b %y'.format(invc.invoice_date) }}
will not work for the life of me - I don't quite get the inconsitency. I also had problems with a Numeric datatype, and had to convert it to a float to get the "".format to work...
Thank you
As the documentation states, it does work.
I had forgotten to wrap the {}placeholder in the actual string,
so its
{{ '{:%d %b %y}'.format(invc.invoice_date) }}
Thank you Raja for inspiring comments :P