How to store websockets at the server side with aiohttp? - python-3.x

I'm trying to make simple web-chat with rooms using aiohttp. Can you please advise me how to store my websockets connections? Some code below are simplified a bit. I'm getting an EOF error from socket time by time (and I can reproduce it), but i don't know why. So, i got a question, am i do it right? Should i close websockets everytime when i reload or follow to link? If not, so, how i will connect client with my already opened socket? Sorry for my eng ^^ thanks.
app.py
import asyncio
import aiohttp_jinja2
import jinja2
import hashlib
import collections
import os
from aiohttp_session import session_middleware
from aiohttp_session.cookie_storage import EncryptedCookieStorage
from aiohttp import web
from routes import routes
from middlewares import authorize
from motor import motor_asyncio as ma
from settings import *
basedir = os.path.dirname(os.path.realpath(__file__))
photo_dir = os.path.join(basedir, 'static/photo/')
async def on_shutdown(app):
for ws in app['websockets']:
await ws.close(code=1001, mesage='Server shutdown')
middle = [
session_middleware(EncryptedCookieStorage(hashlib.sha256(bytes(SECRET_KEY, 'utf-8')).digest())),
authorize
]
app = web.Application(middlewares=middle)
aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader('templates'))
for route in routes:
app.router.add_route(*route[:3], name=route[3])
app['static_root_url'] = '/static'
app.router.add_static('/static', 'static', name='static')
app.client = ma.AsyncIOMotorClient(MONGO_HOST)
app.db = app.client[MONGO_DB_NAME]
app.on_cleanup.append(on_shutdown)
app['websockets'] = collections.defaultdict(list)
app['online'] = {}
app['photo_dir'] = photo_dir
web.run_app(app)
and websocket handler
class CompanyWebSocket(web.View):
async def get(self):
ws = web.WebSocketResponse()
await ws.prepare(self.request)
session = await get_session(self.request)
self_id = session.get('user')
login = session.get('login')
company_id = self.request.rel_url.query.get('company_id')
message = Message(self.request.app.db)
company = Company(self.request.app.db)
my_companys = await company.get_company_by_user(self_id)
for c in my_companys:
self.request.app['websockets'][str(c['_id'])].append(ws)
async for msg in ws:
if msg.type == WSMsgType.TEXT:
if msg.data == 'close':
await ws.close()
else:
await message.save_for_company({'data': 'data'})
mess = {
'data': 'data'
}
# send mess to users in company
for company_ws in self.request.app['websockets'][company_id]:
await company_ws.send_json(mess)
elif msg.type == WSMsgType.ERROR:
log.debug('ws connection closed with exception %s' % ws.exception())
try:
self.request.app['websockets'][company_id].remove(ws)
except:
pass
for _ws in self.request.app['websockets'][company_id]:
await _ws.send_json({'user': login, 'type': 'left'})
return ws

Related

impossible to execute due to infinite loop

the problem itself is that the last 4 lines cannot be executed because there is an infinite loop in the function above, without which it is impossible, who can offer a solution?
from aiogram import Bot, types
from aiogram.dispatcher import Dispatcher
from aiogram.utils import executor
from requests import Request, Session
import requests
import json
import pprint
import time
bot = Bot(token='')
dp = Dispatcher(bot)
site = "https://pro-api.coinmarketcap.com/v2/cryptocurrency/quotes/latest"
parameters = {
'slug':'bitcoin',
'convert':'USD'
}
headers = {
'Accepts':'application/json',
'X-CMC_PRO_API_KEY':''
}
def main():
bitcoin_history = []
while True:
session = Session()
session.headers.update(headers)
response = session.get(site, params=parameters)
price = (json.loads(response.text)['data']['1']['quote']['USD']['price'])
bitcoin_history.append(price)
print(bitcoin_history)
if len(bitcoin_history) == 5:
bitcoin_history = []
time.sleep(30)
main()
#dp.message_handler(commands=["btcusd"])
async def echo_send(message : types.Message):
await message.answer("$" + str())
executor.start_polling(dp, skip_updates=True)

Deployment options for a Flask/PyMongo Discord Bot?

I've seen many tutorials on how to deploy just a Discord Bot or just a basic Flask Web App, however, this project packages both together where the Bot relies completely on interfacing with a REST api.
Presentation Layer: Discord UI. Handles user command input and bot output
Application Layer: Discord handlers that interface with and formats output based on data received from requests (bot.py)
Business Logic Layer: Flask interface that handles data from arguments passed from Discord user input. Handles CRUD requests (app.py)
Data Access Layer: MongoDB database
I currently have it set up to run the app using the routine in this answer for threading: run discord bot inside flask
$ python3 app.py
I've tried uploading the project to a google VM which works but when I exit, it goes offline. I've also tried following a digital ocean tutorial on Dockerizing a Flask and MongoDB app to deploy it to the cloud but it throws errors like
File "/var/www/wsgi.py", line 1, in <module>
flask | from app import app
flask | File "/var/www/app.py", line 5
flask | from import client
flask | ^
flask | SyntaxError: invalid syntax
even though there is no such error...
db.py
from flask import Flask
import pymongo
from pymongo import MongoClient
from app import app
CONNECTION_STRING = "mongodb+srv://<user>:<password>#cluster0.43ypd.mongodb.net/book-bot?retryWrites=true&w=majority"
client = MongoClient(CONNECTION_STRING)
db = client["book-bot"]
books_collection = db["books"]
app.py
from flask import Flask, render_template, request
from flask_mail import *
from random import *
from json import dumps, loads
from bot import client
from threading import Thread
from functools import partial
import time
import os
import db
from book import Book, Seller
app = Flask(__name__)
app.config["MAIL_SERVER"]='smtp.gmail.com'
app.config["MAIL_PORT"] = 465
app.config["MAIL_USERNAME"] = 'email#gmail.com'
app.config['MAIL_PASSWORD'] = 'password'
app.config['MAIL_USE_TLS'] = False
app.config['MAIL_USE_SSL'] = True
mail = Mail(app)
otp = randint(000000,999999)
# localhost:8000/book?course=<course>&number=<course-number>
#app.route('/book/<department>/<course_num>', methods=['GET'])
def book(department, course_num):
query = {"department": department, "course": course_num}
document = db.db.books_collection.find(query, {"_id": 0})
for book in document:
return dumps(book)
return dumps(None)
#app.route('/book/insert/<dep>/<cnum>/<name>/<link>/<buy_price>/<rent_price>', methods=["POST"])
def insert_book(dep, cnum, name, link, buy_price, rent_price):
# Initialize book with the GMU bookstore seller information
seller = Seller("GMU Bookstore", link, float(buy_price), float(rent_price), "Fairfax Campus", True)
# Initialize seller and book data with request params and seller data
bookstore_dict = {"name": seller.name, "link": seller.link,
"buy": seller.buy,
"buy_price": seller.buy_price,
"rent": seller.rent,
"rent_price": seller.rent_price,
"location": seller.location,
"verified": seller.verified
}
book = Book(name, dep, cnum, bookstore_dict)
# create dict struct to insert proper format into Mongo collection
book_dict = {
"name": book.name,
"department": book.department,
"course": book.course_name,
"sellers": book.sellers
}
# Insert book into Database
db.db.books_collection.insert_one(book_dict)
return f"Book name: {book.name} | Department: {book.department} | Course: {book.course_name}"
# example: localhost:8000/book/insert/seller/Mostafa/mostaf#gmu.edu/true/false/Faifax
# "http://localhost:8000/book/insert/seller/{dep}/{cnum}/{name}/{link}/{buy_price}/{rent_price}/{location}
#app.route('/book/insert/seller/<dep>/<cnum>/<name>/<link>/<buy_price>/<rent_price>/<location>', methods=["POST"])
def insert_seller(dep, cnum, name, link, buy_price, rent_price, location, methods=["POST"]):
# temporary solution to interrupted POST request due to # tag in string passed into request
name_arg = name.split("#")
name = name_arg[0] + "#" + name_arg[1]
seller = Seller(name, link, float(buy_price), float(rent_price), location, False)
seller_dict = {
"name": seller.name,
"link": seller.link,
"buy": seller.buy,
"buy_price": seller.buy_price,
"rent": seller.rent,
"rent_price": seller.rent_price,
"location": seller.location,
"verified": seller.verified,
}
query = {"department": dep, "course": cnum}
book = db.db.books_collection.find(query)
for doc in book:
db.db.books_collection.update_one({"_id": doc["_id"]}, {"$push": {"sellers": seller_dict}})
return "Added seller to book"
#app.route('/verify/<email>/<dep>/<cnum>', methods=["POST"])
def verify(email, dep, cnum):
send_otp = f"{otp}-{dep}-{cnum}"
msg = Message('Verify Student Seller', sender='email#gmail.com', recipients=[email])
msg.body = f"Please click on the link to verify your student seller status\n {send_otp}\nhttp://localhost:8000/"
mail.send(msg)
return "Success"
#app.route('/', methods=["GET"])
def load_validation():
return render_template("email.html")
#app.route('/validate', methods=["POST"])
def validate():
user_email = request.form['email']
user_otp = request.form['otp']
args = user_otp.split('-')
# check if numbervalue is equal to otp generated
if int(args[0]) == otp:
query = {"department": args[1], "course": args[2]}
book = db.db.books_collection.find(query)
for doc in book:
db.db.books_collection.update_one(query, {"$set": {"sellers.$[t].verified": True}},
array_filters=[{"t.link": user_email}])
return "<h3>Your seller status has been activated.</h3>"
return "<h3>failure, OTP does not match</h3>"
def flask_thread(func):
thread = Thread(target=func)
print('Start Separate Thread From Bot')
thread.start()
def run():
app.run(host='0.0.0.0', port=8000, use_reloader=False)
if __name__ == '__main__':
flask_thread(func=run)
client.run('bot-token', bot=True)
bot.py
import discord
from discord.ext import commands
import os
import requests
import json
import asyncio
client = discord.Client()
# client = commands.Bot(command_prefix="!", case_insensitive=True)
def insert_seller(dep, cnum, link, buy_price, rent_price, location, name):
response = requests.post(f"http://localhost:8000/book/insert/seller/{dep}/{cnum}/{name}/{link}/{buy_price}/{rent_price}/{location}")
return response
def verify_student(email, dep, cnum):
if ('#gmu.edu' in email):
response = requests.post(f"http://localhost:8000/verify/{email}/{dep}/{cnum}")
return response
return "Please enter a valid email"
def get_book(department, course_num):
response = requests.get(f"http://localhost:8000/book/{department}/{course_num}")
if (response == None):
return None
json_data = json.loads(response.text)
return json_data
#client.event
async def on_ready():
print("We have logged in as {0.user}".format(client))
#client.event
async def on_message(message):
if (message.content.startswith("!bot help")):
msg = "To check for a course textbook enter:\n'!bot check <department> <course_num>\n\nTo add yourself as a seller enter:\n'!bot add <department> <course_num> <your_name> <gmu_email> <buy_price> <rent_price> <city>'"
await message.channel.send(msg)
if (message.content.startswith('!bot check')):
args = message.content.split()
book = get_book(args[2], args[3])
embed = discord.Embed(title=f"{book['department']} {book['course']} Textbook", description=f"\n{book['name']}\n", color=0xFFD700)
for i in book['sellers']:
# hyperlink the bookstore link because it's long
if i['buy']:
options = "buy"
if i['rent']:
options += "/rent"
if i['name'] == "GMU Bookstore":
embed.add_field(name=f"{i['name']}", value=f"[Bookstore link]({i['link']})\nOptions: {options}\nLocation: {i['location']}",inline=False)
else:
embed.add_field(name=f"{i['name']} ({i['link']})", value=f"Options: {options}\nLocation: {i['location']}",inline=False)
embed.set_author(name=message.author.display_name, icon_url=message.author.avatar_url)
embed.set_footer(text="Powered by students. This is not an official GMU service.")
await message.channel.send(embed=embed)
if (message.content.startswith('!bot add')):
args = message.content.split()
book = get_book(args[2], args[3])
if (book == None):
await message.channel.send("Course requirements not found.")
return
def check(msg):
return msg.author == message.author and 'Y' in msg.content
embed = discord.Embed(title=f"Is this your book (enter Y/N)?\n\n", description=f"\n{book['name']}\n", color=0xFFD700)
await message.channel.send(embed=embed)
try:
msg = await client.wait_for("message", check=check, timeout=30)
except asyncio.TimeoutError:
await message.channel.send("Sorry, you didn't reply in time.")
if (msg.content):
# name will save the seller with their discord user handle
name = f"{message.author}"
# Request will 404 because of '#' symbol
regs = name.split('#')
insert_seller(args[2], args[3], args[4], args[5], args[6], args[7], regs[0] + "#" + regs[1])
verify_student(args[4], args[2], args[3])
await message.channel.send("Sending verification email!")
else:
await message.channel.send("Sorry you can't add that.")

Telegram API Bot to invite subscribers of other channels to my own

I want to invite the members of other telegram channels to my channel, but getting this error
THE Error is:
raise ConnectionError('Connection to Telegram failed {} time(s)'.format(self._retries))
ConnectionError: Connection to Telegram failed 5 time(s)
I do not know why I am getting this error, I have checked it with a proxy but it remained the same.
If anyone of you know about this type of error please let me know. I have no idea about this. I do know about this. This is basically a telegram api bot that I want to be completely done with no errors.
import configparser
import json
import asyncio
import sys
import socks
from telethon import TelegramClient
from telethon.tl.types import InputPhoneContact
from telethon.tl.functions.contacts import ImportContactsRequest
from telethon.errors import FloodWaitError, RPCError
#import proxy
from telethon import TelegramClient
from telethon.errors import SessionPasswordNeededError
from telethon.tl.functions.channels import GetParticipantsRequest
from telethon.tl.types import ChannelParticipantsSearch
from telethon.tl.types import (
PeerChannel
)
# Reading Configs
config = configparser.ConfigParser()
config.read("config.ini")
# Setting configuration values
api_id = 1234567
api_hash = 'abcdefghjklmnopqrstuvwxyz1234567'
phone = '+123456789000'
username = '#abcdef'
#proxy='180.179.98.22:3128'
# Create the client and connect
client = TelegramClient(username, api_id, api_hash, proxy=(socks.SOCKS5, '166.62.80.198', 18726))
async def main(phone):
await client.start()
print("Client Created")
# Ensure you're authorized
if await client.is_user_authorized() == False:
await client.send_code_request(phone)
try:
await client.sign_in(phone, input('Enter the code: '))
except SessionPasswordNeededError:
await client.sign_in(password=input('Password: '))
me = await client.get_me()
user_input_channel = input("enter entity(telegram URL or entity id):")
if user_input_channel.isdigit():
entity = PeerChannel(int(user_input_channel))
else:
entity = user_input_channel
my_channel = await client.get_entity(entity)
offset = 0
limit = 100
all_participants = []
while True:
participants = await client(GetParticipantsRequest(
my_channel, ChannelParticipantsSearch(''), offset, limit,
hash=0
))
if not participants.users:
break
all_participants.extend(participants.users)
offset += len(participants.users)
all_user_details = []
for participant in all_participants:
all_user_details.append(
{"id": participant.id, "first_name": participant.first_name, "last_name": participant.last_name,
"user": participant.username, "phone": participant.phone, "is_bot": participant.bot})
with open('user_data.json', 'w') as outfile:
json.dump(all_user_details, outfile)
with client:
client.loop.run_until_complete(main(phone))

Python gRPC service for Envoy ratelimit

I am trying to create a small service to respond to Envoy's rate limiting queries. I have compiled all the relevant protobuff files and the one relevant for the service I am trying to implement is here:
https://github.com/envoyproxy/envoy/blob/v1.17.1/api/envoy/service/ratelimit/v3/rls.proto
There is a service definition in there but inside of the "compiled" python file, all I see about it is this:
_RATELIMITSERVICE = _descriptor.ServiceDescriptor(
name='RateLimitService',
full_name='envoy.service.ratelimit.v3.RateLimitService',
file=DESCRIPTOR,
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_start=1531,
serialized_end=1663,
methods=[
_descriptor.MethodDescriptor(
name='ShouldRateLimit',
full_name='envoy.service.ratelimit.v3.RateLimitService.ShouldRateLimit',
index=0,
containing_service=None,
input_type=_RATELIMITREQUEST,
output_type=_RATELIMITRESPONSE,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
])
_sym_db.RegisterServiceDescriptor(_RATELIMITSERVICE)
DESCRIPTOR.services_by_name['RateLimitService'] = _RATELIMITSERVICE
here is my feeble attempt at implementing the service
import logging
import asyncio
import grpc
from envoy.service.ratelimit.v3.rls_pb2 import RateLimitResponse, RateLimitRequest
class RL:
def ShouldRateLimit(self, request):
result = RateLimitResponse()
def add_handler(servicer, server):
rpc_method_handlers = {
'ShouldRateLimit': grpc.unary_unary_rpc_method_handler(
RL.ShouldRateLimit,
request_deserializer=RateLimitRequest.FromString,
response_serializer=RateLimitResponse.SerializeToString,
)
}
generic_handler = grpc.method_handlers_generic_handler(
'envoy.service.ratelimit.v3.RateLimitService',
rpc_method_handlers
)
server.add_generic_rpc_handlers((generic_handler,))
async def serve():
server = grpc.aio.server()
add_handler(RL(), server)
listen_addr = '[::]:5051'
server.add_insecure_port(listen_addr)
logging.info(f'Starting server on {listen_addr}')
await server.start()
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
asyncio.run(serve())
How am I supposed to return (or even instantiate) a RateLimitResponse back to the caller ?

How do I receive new responses in the Websocket?

I have a client module:
#!/usr/bin/env python
# WS client example
import asyncio
import websockets
async def hello():
async with websockets.connect(
'ws://A.B.C.D:8765') as websocket:
name = input("What's your name? ")
await websocket.send(name)
greeting = await websocket.recv()
print(greeting)
asyncio.get_event_loop().run_until_complete(hello())
and the server module:
from __future__ import print_function
#!/usr/bin/env python
import asyncio
import datetime
import random
import websockets
import ast
from collections import defaultdict
import csv
import datetime
from itertools import chain
import json
import os
import operator
import sys
import pymongo
from pymongo import MongoClient
try:
client = MongoClient('localhost', 27017)
db = client["Bubble"]
except Exception as e:
print(e)
start_match = datetime.datetime.strptime(
"2018-07-01 18:00:00", '%Y-%m-%d %H:%M:%S')
collection = "CRODEN_R16"
async def hello(websocket, path):
entity_name = await websocket.recv()
print(entity_name)
while True:
file = open("set_start_match.txt", "r")
for line in file:
start_today = datetime.datetime.strptime(
line.split('.')[0], '%Y-%m-%d %H:%M:%S')
print(start_today)
now = datetime.datetime.utcnow()
diff = now - start_today
request_match = start_match + diff
print(diff)
for post in db[collection].find():
if "emotion" not in post.keys():
print("Ignored")
continue
if post["timeStamp"] > request_match:
if post["entity_name"] == entity_name:
print("Satisfied")
currDict = {}
currDict["entity"] = post["entity_name"]
currDict["emotion"] = max(
post["emotion"].items(), key=operator.itemgetter(1))[0]
currDict["profile_image"] = post["userProfile"]
currDict["tweet"] = post["tweet"]
currDict_json = json.dumps(currDict, default=str)
print(currDict["tweet"])
await websocket.send(currDict_json)
await asyncio.sleep(1)
del currDict
try:
start_server = websockets.serve(hello, '0.0.0.0', 8765)
print("Start entity server")
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
except Exception as e:
print(e)
Now, the issue is that I want to send name as an input only once and receive the output continuously.
When I wrote this in client:
while True:
greeting = await.websocket.recv()
print(greeting)
The same response is returned again and again. Even on the server side, where I am printing the rendered results from db, I am printing the same doc.
I am completely clueless as to what is the issue?
Note: I have tried to the run the once-run client module and there I was getting perfect results. It was just that I had to give the same input again and again. I want it to be automated.
To get data continuously someone has to send data continuously.
If someone sends data continuously then someone else has to get data continuously.
So both sides need loop.
client - it sends numbers continuously in loop.
#!/usr/bin/env python
import asyncio
import websockets
import time
async def hello():
async with websockets.connect(
'ws://localhost:8769') as websocket:
name = input("What's your name? ")
await websocket.send(name)
i = 0
while True:
print('send:', i)
await websocket.send(str(i))
time.sleep(2)
i += 1
try:
asyncio.get_event_loop().run_until_complete(hello())
except KeyboardInterrupt:
print('KeyboardInterrupt')
server - it receives numbers continuously in loop
import asyncio
import websockets
async def hello(websocket, path):
entity_name = await websocket.recv()
print('name:', entity_name)
while True:
data = await websocket.recv()
print('recv:', data)
try:
print("Start entity server")
start_server = websockets.serve(hello, '0.0.0.0', 8769)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
except KeyboardInterrupt: # keyboard need special except
print("KeyboardInterrupt")
start_server.ws_server.close() # solutin for [Errno 98]
except Exception as ex:
print(ex)

Resources