updates database design for mariadb

This commit is contained in:
Raphael Maenle 2020-05-27 20:06:43 +02:00
parent 0cfc801f59
commit 355ba99ca3
18 changed files with 435 additions and 139 deletions

@ -1 +1 @@
Subproject commit 26cc3425ee50ff516d75903d9e281b30860c497b
Subproject commit fe1c216c292b1e9c680e2be33103a9439cba5e03

View File

@ -3,6 +3,5 @@ basedir = os.path.abspath(os.path.dirname(__file__))
class Config(object):
# ...
SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') or \
'sqlite:///' + os.path.join(basedir, 'app.db')
SQLALCHEMY_DATABASE_URI = 'mysql://user:pw@mariadb:3306/calendarwatch'
SQLALCHEMY_TRACK_MODIFICATIONS = False

View File

@ -0,0 +1 @@
Generic single-database configuration.

View File

@ -0,0 +1,45 @@
# A generic, single database configuration.
[alembic]
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

View File

@ -0,0 +1,96 @@
from __future__ import with_statement
import logging
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
logger = logging.getLogger('alembic.env')
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
from flask import current_app
config.set_main_option(
'sqlalchemy.url',
str(current_app.extensions['migrate'].db.engine.url).replace('%', '%%'))
target_metadata = current_app.extensions['migrate'].db.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

View File

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View File

@ -0,0 +1,34 @@
"""empty message
Revision ID: 1e8205594ac1
Revises: aeab4aff199b
Create Date: 2020-05-27 16:57:54.384047
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '1e8205594ac1'
down_revision = 'aeab4aff199b'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('user', sa.Column('userid', sa.String(length=64), nullable=True))
op.create_index(op.f('ix_user_userid'), 'user', ['userid'], unique=True)
op.drop_index('ix_user_username', table_name='user')
op.create_index(op.f('ix_user_username'), 'user', ['username'], unique=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_user_username'), table_name='user')
op.create_index('ix_user_username', 'user', ['username'], unique=True)
op.drop_index(op.f('ix_user_userid'), table_name='user')
op.drop_column('user', 'userid')
# ### end Alembic commands ###

View File

@ -0,0 +1,23 @@
"""empty message
Revision ID: 3b829a27bc337
Revises: 1e8205594ac1
Create Date: 2020-05-27 19:30:54.384047
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '3b829a27bc337'
down_revision = '1e8205594ac1'
branch_labels = None
depends_on = None
def upgrade():
op.alter_column('User', 'google_credentials', existing_type=sa.Relationship(), new_column_name='google_token')
def downgrade():
op.alter_column('User', 'google_token', existing_type=sa.Relationship(), new_column_name='google_credentials')

View File

@ -0,0 +1,72 @@
"""empty message
Revision ID: aeab4aff199b
Revises:
Create Date: 2020-05-27 15:23:20.611265
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'aeab4aff199b'
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('user',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('username', sa.String(length=64), nullable=True),
sa.Column('email', sa.String(length=120), nullable=True),
sa.Column('profile_pic', sa.String(length=256), nullable=True),
sa.Column('password_hash', sa.String(length=128), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_user_email'), 'user', ['email'], unique=True)
op.create_index(op.f('ix_user_username'), 'user', ['username'], unique=True)
op.create_table('calendar',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('calendar_id', sa.String(length=256), nullable=False),
sa.Column('name', sa.String(length=256), nullable=True),
sa.Column('toggle', sa.String(length=8), nullable=True),
sa.Column('color', sa.String(length=16), nullable=True),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id', 'calendar_id')
)
op.create_index(op.f('ix_calendar_name'), 'calendar', ['name'], unique=False)
op.create_index(op.f('ix_calendar_user_id'), 'calendar', ['user_id'], unique=False)
op.create_table('device',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('deviceName', sa.String(length=64), nullable=True),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('deviceName')
)
op.create_table('google_token',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('token', sa.String(length=256), nullable=True),
sa.Column('refresh_token', sa.String(length=256), nullable=True),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('google_token')
op.drop_table('device')
op.drop_index(op.f('ix_calendar_user_id'), table_name='calendar')
op.drop_index(op.f('ix_calendar_name'), table_name='calendar')
op.drop_table('calendar')
op.drop_index(op.f('ix_user_username'), table_name='user')
op.drop_index(op.f('ix_user_email'), table_name='user')
op.drop_table('user')
# ### end Alembic commands ###

View File

@ -8,13 +8,15 @@ def load_user(id):
return User.query.get(id)
class User(UserMixin, db.Model):
id = db.Column(db.String(64), primary_key=True)
username = db.Column(db.String(64), index=True, unique=True)
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
userid = db.Column(db.String(64), index=True, unique=True)
username = db.Column(db.String(64), index=True)
email = db.Column(db.String(120), index=True, unique=True)
profile_pic = db.Column(db.String(256))
password_hash = db.Column(db.String(128))
calendarJson = db.Column(db.String)
google_credentials = db.Column(db.String)
google_token = db.relationship('GoogleToken', uselist=False, backref = 'user')
calendars = db.relationship('Calendar', backref='user', lazy=True)
devices = db.relationship('Device', backref='user')
def __repr__(self):
return '<User {}>'.format(self.username)
@ -25,53 +27,11 @@ class User(UserMixin, db.Model):
def checkPassword(self, password):
return check_password_hash(self.password_hash, password)
def setJson(self, jsonObject):
self.calendarJson = json.dumps(jsonObject)
db.session.commit()
def getJson(self):
return json.loads(self.calendarJson)
def setGoogleCredentials(self, credentials):
self.google_credentials = json.dumps(credentials)
db.session.commit()
def getGoogleCredentials(self):
if self.google_credentials is None:
print("no credentials", flush=True)
return None
return json.loads(self.google_credentials)
class Calendar(db.Model):
usr_id = db.Column(db.String(21), index=True)
calendar_id = db.Column(db.String(256), primary_key=True)
name = db.Column(db.String(256), index=True)
toggle = db.Column(db.String(8))
color = db.Column(db.String(16))
def getCalendars(self, user_id):
calendars = self.query.filter(Calendar.usr_id==user_id)
return calendars
def getCalendar(self, user_id, calendar_id):
calendars = self.query.filter(self.usr_id==user_id, self.calendar_id==calendar_id)
calendar = None
for c in calendars:
calendar = c
if not calendar:
return None
return calendar
@staticmethod
def updateCalendar(user_id, calendar_id, toggle=None, color=None):
calendar = Calendar.query.filter(Calendar.usr_id==user_id, Calendar.calendar_id==calendar_id).first()
def updateCalendar(self, calendar_id, toggle=None, color=None):
for calendar in self.calendars:
if calendar.calendar_id == calendar_id:
break
print("updating", flush=True)
if(toggle != None):
@ -83,12 +43,28 @@ class Calendar(db.Model):
calendar.color = color
db.session.commit()
def create(self, user_id, calendar_id, name, color, toggle = 'True'):
newcal = Calendar(usr_id=user_id, calendar_id=calendar_id, name=name, toggle=toggle, color=color)
def hasCalendar(self, calendar_id):
for calendar in self.calendars:
if calendar.calendar_id == calendar_id:
return True
db.session.add(newcal)
db.session.commit()
return False
class GoogleToken(db.Model):
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
token = db.Column(db.String(256))
refresh_token = db.Column(db.String(256))
class Device(db.Model):
id = db.Column(db.Integer, primary_key=True)
deviceId = db.Column(db.String(128), index=True)
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
deviceName = db.Column(db.String(64), unique=True)
class Calendar(db.Model):
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
user_id = db.Column(db.Integer, db.ForeignKey('user.id'), index=True, nullable=False)
calendar_id = db.Column(db.String(256), primary_key=True)
name = db.Column(db.String(256), index=True)
toggle = db.Column(db.String(8))
color = db.Column(db.String(16))

Binary file not shown.

View File

@ -1,13 +1,11 @@
FROM python:3.8-slim-buster
RUN apt-get update && apt-get upgrade
RUN apt-get install -y cron
RUN pip3 install flask Flask-SQLAlchemy flask_migrate flask_login flask_wtf python-dotenv
RUN apt-get install gcc libpcre3 libpcre3-dev -y
RUN apt-get install gcc libpcre3 libpcre3-dev libmariadbclient-dev -y
RUN pip3 install uwsgi
RUN pip3 install email-validator
RUN pip3 install google google-oauth google-auth-oauthlib google-api-python-client
RUN pip3 install google google-oauth google-auth-oauthlib google-api-python-client mysqlclient
COPY docker-entrypoint.sh /usr/local/bin/
EXPOSE 8084
EXPOSE 3001
ENTRYPOINT ["docker-entrypoint.sh"]

View File

@ -1,7 +1,7 @@
#!/bin/sh
cd /home/calendarwatch
# uwsgi --http-socket 0.0.0.0:8084 -w wsgi --protocol=https
export FLASK_APP=/home/calendarwatch/server.py
python3 server.py
echo "server has been started"

View File

@ -1,11 +1,29 @@
version: '3'
services:
calendarwatch:
build:
context: ./calendarwatch
image: calendarwatch:latest
container_name: calendarwatch
volumes:
- ../:/home/calendarwatch
ports:
- "0.0.0.0:8084:8084"
calendarwatch:
build:
context: ./calendarwatch
image: calendarwatch:latest
container_name: calendarwatch
environment:
- FLASK_APP=/home/calendarwatch/server.py
volumes:
- ../:/home/calendarwatch
ports:
- "0.0.0.0:8084:8084"
mariadb:
image: mariadb
container_name: maridab
environment:
- MYSQL_ROOT_PASSWORD=pw
- MYSQL_DATABASE=calendarwatch
- MYSQL_USER=user
- MYSQL_PASSWORD=pw
volumes:
- database:/var/lib/mysql
volumes:
database:
driver: local

View File

@ -13,7 +13,6 @@ app = Flask(__name__,
static_folder='static',
template_folder='template')
app.secret_key = os.environ.get("SECRET_KEY") or os.urandom(24)
app.config.from_object(Config)
db = SQLAlchemy(app)

View File

@ -9,7 +9,6 @@ import flask
# Python standard libraries
import json
import os
import sqlite3
# Third-party libraries
import flask
@ -24,31 +23,47 @@ from flask_login import (
import requests
from database.models import Calendar as dbCalendar
from server import db
# Configuration
CLIENT_SECRETS_FILE = "certificate/client_secret.json"
class GoogleClient():
def __init__(self):
self.CLIENT_SECRETS_FILE = "certificate/client_secret.json"
# This OAuth 2.0 access scope allows for full read/write access to the
# authenticated user's account and requires requests to use an SSL connection.
SCOPES = ["https://www.googleapis.com/auth/userinfo.email", "https://www.googleapis.com/auth/userinfo.profile", "https://www.googleapis.com/auth/calendar.readonly", "openid"]
API_SERVICE_NAME = 'calendar'
API_VERSION = 'v3'
with open("/home/calendarwatch/certificate/google_client.json", encoding='utf-8') as json_file:
self.google_client = json.load(json_file)
GOOGLE_CLIENT_ID ="377787187748-shuvi4iq5bi4gdet6q3ioataimobs4lh.apps.googleusercontent.com"
GOOGLE_CLIENT_SECRET = "Hu_YWmKsVKUcLwyeINYzdKfZ"
GOOGLE_DISCOVERY_URL = (
"https://accounts.google.com/.well-known/openid-configuration"
)
self.SCOPES = self.google_client.get('scopes')
self.API_SERVICE_NAME = 'calendar'
self.API_VERSION = 'v3'
# OAuth 2 client setup
client = WebApplicationClient(GOOGLE_CLIENT_ID)
# GOOGLE_CLIENT_ID ="377787187748-shuvi4iq5bi4gdet6q3ioataimobs4lh.apps.googleusercontent.com"
self.GOOGLE_CLIENT_ID = self.google_client.get('client_id')
# GOOGLE_CLIENT_SECRET = "Hu_YWmKsVKUcLwyeINYzdKfZ"
self.GOOGLE_CLIENT_SECRET = self.google_client.get('client_secret')
self.GOOGLE_DISCOVERY_URL = (
"https://accounts.google.com/.well-known/openid-configuration"
)
# OAuth 2 client setup
self.client = WebApplicationClient(self.GOOGLE_CLIENT_ID)
def build_credentials(self, token, refresh_token):
data = {}
data['token'] = token
data['refresh_token'] = refresh_token
data['token_uri'] = self.google_client.get('token_uri')
data['client_id'] = self.google_client.get('client_id')
data['client_secret'] = self.google_client.get('client_secret')
data['scopes'] = self.google_client.get('scopes')
return data
GC = GoogleClient()
def login():
# Create flow instance to manage the OAuth 2.0 Authorization Grant Flow steps.
flow = google_auth_oauthlib.flow.Flow.from_client_secrets_file(
CLIENT_SECRETS_FILE, scopes=SCOPES)
GC.CLIENT_SECRETS_FILE, scopes=GC.SCOPES)
# The URI created here must exactly match one of the authorized redirect URIs
# for the OAuth 2.0 client, which you configured in the API Console. If this
# value doesn't match an authorized URI, you will get a 'redirect_uri_mismatch'
@ -72,7 +87,7 @@ def verifyResponse():
state = flask.session['state']
flow = google_auth_oauthlib.flow.Flow.from_client_secrets_file(
CLIENT_SECRETS_FILE, scopes=SCOPES, state=state)
GC.CLIENT_SECRETS_FILE, scopes=GC.SCOPES, state=state)
flow.redirect_uri = flask.url_for('callback', _external=True)
# Use the authorization server's response to fetch the OAuth 2.0 tokens.
@ -91,11 +106,11 @@ def verifyResponse():
def get_google_provider_cfg():
return requests.get(GOOGLE_DISCOVERY_URL).json()
return requests.get(GC.GOOGLE_DISCOVERY_URL).json()
def deleteAccount(user):
result = requests.post('https://oauth2.googleapis.com/revoke',
params={'token': user.get('token')},
params={'token': user.google_token.token},
headers = {'content-type': 'applixation/x-www-form-urlencoded'})
print(result, flush=True)
return
@ -107,10 +122,10 @@ class Calendar:
self.toggle=toggle
self.calendarId = calendarId
# TODO move this to databas
def calendarsFromDb():
calendars = dbCalendar.getCalendars(dbCalendar, current_user.id)
pyCalendars = []
for calendar in calendars:
for calendar in current_user.calendars:
name = (calendar.name[:16] + '..') if len(calendar.name)> 18 else calendar.name
calendarId = calendar.calendar_id
toggle = calendar.toggle
@ -121,20 +136,6 @@ def calendarsFromDb():
return pyCalendars
def getCalendarJson():
if 'credentials' not in flask.session:
return flask.redirect('login/google')
# Load credentials from the session.
credentials = google.oauth2.credentials.Credentials(
**flask.session['credentials'])
with open('./userinfo/' + current_user.id + '/calendarevents.json', 'w') as outfile:
json.dump(todaysCal, outfile)
return todaysCal
def updateCalendars():
if 'credentials' not in flask.session:
return flask.redirect('login/google')
@ -145,23 +146,36 @@ def updateCalendars():
# a = flask.session['credentials']
# print(a, flush=True)
# print(current_user.getGoogleCredentials(), flush=True)
if current_user.getGoogleCredentials() == None:
if current_user.google_token == None:
print("notok", flush=True)
return
credentials = google.oauth2.credentials.Credentials(**current_user.getGoogleCredentials())
client_token = GC.build_credentials(current_user.google_token.token,
current_user.google_token.refresh_token)
credentials = google.oauth2.credentials.Credentials(**client_token)
calendars = caltojson.getCalendarList(credentials)
print(calendars, flush=True)
for calendar in calendars:
print(calendar, flush=True)
if not current_user.hasCalendar(calendar.calendarId):
print("adding", flush=True)
c = dbCalendar(calendar_id=calendar.calendarId,
name = calendar.summary,
toggle = "False",
color = calendar.color)
db.session.add(c)
current_user.calendars.append(c)
if dbCalendar.getCalendar(dbCalendar, current_user.id, calendar.calendarId) == None:
dbCalendar.create(dbCalendar, current_user.id, calendar.calendarId, calendar.summary, calendar.color)
db.session.commit()
print("updated Calendars")
# Save credentials back to session in case access token was refreshed.
# ACTION ITEM: In a production app, you likely want to save these
# credentials in a persistent database instead.
# TODO add save updated token to database here
flask.session['credentials'] = credentials_to_dict(credentials)
def credentials_to_dict(credentials):
return {'token': credentials.token,
'refresh_token': credentials.refresh_token,

View File

@ -21,7 +21,7 @@ import server.googleHandler as google
from backend.Routine import Routine
from server import login_manager, app, db
from server.forms import LoginForm, RegistrationForm, DeviceForm
from database.models import User, Calendar, Device
from database.models import User, Calendar, Device, GoogleToken
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
@ -105,19 +105,18 @@ def register():
def deleteAccount():
if not current_user.is_authenticated:
return redirect(url_for('account'))
print(current_user.getGoogleCredentials(), flush=True)
google.deleteAccount(current_user.getGoogleCredentials())
# TODO fix google delete account
google.deleteAccount(current_user)
user = db.session.query(User).filter(User.id==current_user.id).first()
logout_user()
db.session.delete(user)
db.session.delete(current_user)
db.session.commit()
logout_user()
return redirect(url_for('account'))
@app.route("/login/google")
def googlelogin():
if current_user.is_authenticated and current_user.getGoogleCredentials() != None:
if current_user.is_authenticated and current_user.google_credentials.refresh_token != None:
return redirect(url_for('account'))
authorization_url = google.login()
@ -127,35 +126,33 @@ def googlelogin():
@app.route("/login/google/callback")
def callback():
session, credentials = google.verifyResponse()
if current_user.is_authenticated and current_user.getGoogleCredentials == None:
current_user.setGoogleCredentials(credentials)
userinfo = session.get('https://www.googleapis.com/userinfo/v2/me').json()
# Create a user in your db with the information provided
# by Google
# Doesn't exist? Add it to the database.
if not User.query.get(userinfo['id']):
if not db.session.query(User).filter(User.userid==userinfo['id']).first():
gc = GoogleToken(token=credentials.get("token"),
refresh_token=credentials.get("refresh_token"))
db.session.add(gc)
newser = User(
id=userinfo['id'],
userid=userinfo['id'],
username=userinfo['name'],
email=userinfo['email'],
profile_pic=userinfo['picture'],
password_hash=""
password_hash="",
google_token = gc
)
db.session.add(newser)
db.session.commit()
user = User.query.get(userinfo['id'])
user = db.session.query(User).filter(User.userid==userinfo['id']).first()
# Begin user session by logging the user in
print("login:" + user.id)
login_user(user)
# TODO currently not using the credentials anymore
if user.getGoogleCredentials() is None:
user.setGoogleCredentials(credentials)
return flask.redirect(flask.url_for('index'))
@app.route("/logout")

Binary file not shown.