masq
/
halfapi
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[conf] use of toml for halfapi configs. re-enable possibility of multiple domains

This commit is contained in:
Maxime Alves LIRMM 2021-12-03 17:25:57 +01:00
parent d06857bf49
commit dbca2f28fb
15 changed files with 210 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Pipfile
View File

@ -21,6 +21,8 @@ pyjwt = ">=2.3.0,<2.4.0"
pyyaml = ">=5.3.1,<6"
timing-asgi = ">=0.2.1,<1"
schema = ">=0.7.4,<1"
toml = "*"
pip = "*"
[scripts]
halfapi = "python -m halfapi"

16
halfapi/cli/domain.py
View File

@ -13,7 +13,7 @@ import orjson
from .cli import cli
from ..conf import config, write_config, DOMAINSDICT
from ..conf import write_config
from ..lib.domain import domain_schema
from ..lib.schemas import schema_dict_dom
@ -61,13 +61,7 @@ def create_domain(domain_name: str, module_path: str):
os.mkdir(router_path)
create_init(router_path)
if not config.has_section('domain'):
config.add_section('domain')
config.set('domain', 'name', domain_name)
config.set('domain', 'router', module_path)
write_config()
# TODO: Generate config file
domain_tree_create()
"""
@ -113,11 +107,13 @@ def list_routes(domain, m_dom):
def list_api_routes():
"""
Echoes the list of all active domains.
TODO: Rewrite function
"""
click.echo('# API Routes')
for domain, m_dom in DOMAINSDICT().items():
list_routes(domain, m_dom)
# for domain, m_dom in DOMAINSDICT().items():
# list_routes(domain, m_dom)
@click.option('--read',default=True, is_flag=True)

19
halfapi/cli/run.py
View File

@ -10,9 +10,10 @@ import uvicorn
from .cli import cli
from .domain import list_api_routes
from ..conf import (PROJECT_NAME, HOST, PORT, SCHEMA,
PRODUCTION, LOGLEVEL, DOMAINSDICT, CONFIG, DOMAIN, ROUTER)
PRODUCTION, LOGLEVEL, CONFIG)
from ..logging import logger
from ..lib.schemas import schema_csv_dict
from ..half_domain import HalfDomain
@click.option('--host', default=HOST)
@click.option('--port', default=PORT)
@ -26,7 +27,8 @@ from ..lib.schemas import schema_csv_dict
@click.argument('schema', type=click.File('r'), required=False)
@click.argument('domain', required=False)
@cli.command()
def run(host, port, reload, secret, production, loglevel, prefix, check, dryrun, schema, domain):
def run(host, port, reload, secret, production, loglevel, prefix, check, dryrun,
schema, domain):
"""
The "halfapi run" command
"""
@ -58,6 +60,19 @@ def run(host, port, reload, secret, production, loglevel, prefix, check, dryrun,
for key, val in schema_csv_dict(schema, prefix).items():
SCHEMA[key] = val
if domain:
# If we specify a domain to run as argument
for key in CONFIG['domain']:
# Disable all domains
CONFIG['domain'].pop(key)
# And activate the desired one, mounted without prefix
CONFIG['domain'][domain] = {
'name': domain,
'prefix': False
}
# list_api_routes()
click.echo(f'uvicorn.run("halfapi.app:application"\n' \

105
halfapi/conf.py
View File

@ -10,7 +10,6 @@ It uses the following environment variables :
It defines the following globals :
- PROJECT_NAME (str) - HALFAPI_PROJECT_NAME
- DOMAINSDICT ({domain_name: domain_module}) - HALFAPI_DOMAIN_NAME / HALFAPI_DOMAIN_MODULE
- PRODUCTION (bool) - HALFAPI_PRODUCTION
- LOGLEVEL (string) - HALFAPI_LOGLEVEL
- BASE_DIR (str) - HALFAPI_BASE_DIR
@ -18,7 +17,6 @@ It defines the following globals :
- PORT (int) - HALFAPI_PORT
- CONF_DIR (str) - HALFAPI_CONF_DIR
- DRYRUN (bool) - HALFAPI_DRYRUN
- config (ConfigParser)
It reads the following ressource :
@ -30,43 +28,41 @@ It follows the following format :
name = PROJECT_NAME
halfapi_version = HALFAPI_VERSION
[domains]
domain_name = requirements-like-url
[domain.domain_name]
name = domain_name
routers = routers
[domain.domain_name.config]
option = Argh
"""
import logging
import os
from os import environ
import sys
from configparser import ConfigParser
import importlib
import tempfile
import uuid
import toml
from .lib.domain import d_domains
from .logging import logger
CONFIG = {}
PROJECT_NAME = environ.get('HALFAPI_PROJECT_NAME') or os.path.basename(os.getcwd())
DOMAINSDICT = lambda: {}
DOMAINS = {}
PRODUCTION = True
LOGLEVEL = 'info'
HOST = '127.0.0.1'
PORT = '3000'
SECRET = ''
CONF_FILE = os.environ.get('HALFAPI_CONF_FILE', '.halfapi/config')
DRYRUN = bool(os.environ.get('HALFAPI_DRYRUN', False))
DOMAIN = None
ROUTER = None
SCHEMA = {}
config = ConfigParser(allow_no_value=True)
CONF_DIR = environ.get('HALFAPI_CONF_DIR', '/etc/half_api')
HALFAPI_ETC_FILE=os.path.join(
CONF_DIR, 'default.ini'
CONF_DIR, 'config'
)
HALFAPI_DOT_FILE=os.path.join(
os.getcwd(), '.halfapi', 'config')
@ -85,59 +81,65 @@ def write_config():
"""
Writes the current config to the highest priority config file
"""
with open(conf_files()[-1], 'w') as halfapi_config:
config.write(halfapi_config)
# with open(conf_files()[-1], 'w') as halfapi_config:
# config.write(halfapi_config)
pass
def config_dict():
"""
The config object as a dict
"""
return {
section: dict(config.items(section))
for section in config.sections()
}
def read_config():
"""
The highest index in "filenames" are the highest priorty
"""
config.read(HALFAPI_CONFIG_FILES)
return toml.load(HALFAPI_CONFIG_FILES)
CONFIG = {}
read_config()
PROJECT_NAME = config.get('project', 'name', fallback=PROJECT_NAME)
PROJECT_NAME = CONFIG.get('project', {}).get(
'name',
environ.get('HALFAPI_PROJECT_NAME', os.path.basename(os.getcwd())))
if len(PROJECT_NAME) == 0:
raise Exception('Need a project name as argument')
if len(CONFIG.get('domain', {}).keys()) == 0:
logger.info('No domains')
# logger.info('Running without domains: %s', d_domains(config) or 'empty domain dictionary')
DOMAINSDICT = lambda: d_domains(config)
DOMAINS = DOMAINSDICT()
if len(DOMAINS) == 0:
logger.info('Running without domains: %s', d_domains(config) or 'empty domain dictionary')
HOST = config.get('project', 'host', fallback=environ.get('HALFAPI_HOST', '127.0.0.1'))
PORT = config.getint('project', 'port', fallback=environ.get('HALFAPI_PORT', '3000'))
# Bind
HOST = CONFIG.get('project', {}).get(
'host',
environ.get('HALFAPI_HOST', '127.0.0.1'))
PORT = int(CONFIG.get('project', {}).get(
'port',
environ.get('HALFAPI_PORT', '3000')))
# Secret
SECRET = CONFIG.get('project', {}).get(
'secret',
environ.get('HALFAPI_SECRET'))
if not SECRET:
# TODO: Create a temporary secret
_, SECRET = tempfile.mkstemp()
with open('SECRET', 'w') as secret_file:
secret_file.write(str(uuid.uuid4()))
secret_path = config.get('project', 'secret', fallback=environ.get('HALFAPI_SECRET', ''))
try:
with open(secret_path, 'r') as secret_file:
SECRET = secret_file.read().strip()
with open(SECRET, 'r') as secret_file:
CONFIG['secret'] = SECRET.strip()
except FileNotFoundError as exc:
logger.info('Running without secret file: %s', secret_path or 'no file specified')
logger.info('Running without secret file: %s', SECRET or 'no file specified')
PRODUCTION = config.getboolean('project', 'production',
fallback=environ.get('HALFAPI_PROD', True))
PRODUCTION = bool(CONFIG.get('project', {}).get(
'production',
environ.get('HALFAPI_PROD', True)))
LOGLEVEL = config.get('project', 'loglevel',
fallback=environ.get('HALFAPI_LOGLEVEL', 'info')).lower()
LOGLEVEL = CONFIG.get('project', {}).get(
'loglevel',
environ.get('HALFAPI_LOGLEVEL', 'info')).lower()
BASE_DIR = config.get('project', 'base_dir',
fallback=environ.get('HALFAPI_BASE_DIR', '.'))
BASE_DIR = CONFIG.get('project', {}).get(
'base_dir',
environ.get('HALFAPI_BASE_DIR', '.'))
CONFIG = {
'project_name': PROJECT_NAME,
@ -146,4 +148,5 @@ CONFIG = {
'host': HOST,
'port': PORT,
'dryrun': DRYRUN,
'domain': {}
}

57
halfapi/half_domain.py Normal file
View File

@ -0,0 +1,57 @@
import importlib
from starlette.applications import Starlette
from starlette.routing import Router
from .half_route import HalfRoute
from .lib.routes import gen_domain_routes, gen_schema_routes
from .lib.domain_middleware import DomainMiddleware
from .logging import logger
class HalfDomain(Starlette):
def __init__(self, app, domain, router=None, config={}):
self.app = app
self.m_domain = importlib.import_module(domain)
self.name = getattr('__name__', domain, domain)
if not router:
self.router = getattr('__router__', domain, '.routers')
else:
self.router = router
self.m_router = importlib.import_module(self.router, domain)
self.m_acl = importlib.import_module(f'{domain}.acl')
self.config = config
"""
if domain:
m_domain = importlib.import_module(domain)
if not router:
router = getattr('__router__', domain, '.routers')
m_domain_router = importlib.import_module(router, domain)
m_domain_acl = importlib.import_module(f'{domain}.acl')
if not(m_domain and m_domain_router and m_domain_acl):
raise Exception('Cannot import domain')
self.schema = domain_schema(m_domain)
routes = [ Route('/', JSONRoute(self.schema)) ]
"""
logger.info('HalfDomain creation %s %s', domain, config)
super().__init__(
routes=gen_domain_routes(self.m_router),
middleware=[
(DomainMiddleware,
{
'domain': self.name,
'config': self.config
}
)
]
)

40
halfapi/halfapi.py
View File

@ -40,13 +40,14 @@ from .lib.domain import domain_schema_dict, NoDomainsException, domain_schema
from .lib.routes import gen_domain_routes, gen_schema_routes, JSONRoute
from .lib.schemas import schema_json, get_acls
from .logging import logger, config_logging
from .half_domain import HalfDomain
from halfapi import __version__
class HalfAPI:
def __init__(self, config,
routes_dict=None):
d_routes=None):
config_logging(logging.DEBUG)
SECRET = config.get('secret')
@ -55,27 +56,28 @@ class HalfAPI:
DRYRUN = config.get('dryrun', False)
self.PRODUCTION = PRODUCTION
self.CONFIG = CONFIG
self.CONFIG = config
self.SECRET = SECRET
self.__application = None
# Domains
""" HalfAPI routes (if not PRODUCTION, includes debug routes)
"""
routes = []
routes.append(
Route('/', JSONRoute({}))
Mount('/halfapi', routes=list(self.halfapi_routes()))
)
routes.append(
Mount('/halfapi', routes=list(self.routes()))
)
logger.info('Config: %s', config)
logger.info('Active domains: %s', config.get('domain', {}))
if routes_dict:
# Mount the routes from the routes_dict argument - domain-less mode
if d_routes:
# Mount the routes from the d_routes argument - domain-less mode
logger.info('Domain-less mode : the given schema defines the activated routes')
for route in gen_schema_routes(routes_dict):
for route in gen_schema_routes(d_routes):
routes.append(route)
else:
"""
@ -104,6 +106,24 @@ class HalfAPI:
on_startup=startup_fcts
)
for key, domain in config.get('domain', {}).items():
dom_name = domain.get('name', key)
if domain.get('prefix', False):
path = f'/{dom_name}'
else:
path = '/'
self.__application.mount(path,
Mount('/',
HalfDomain(
self.application,
domain.get('name', key),
domain.get('router'),
config=domain.get('config', {})
)
)
)
"""
self.__application.add_middleware(
DomainMiddleware,
@ -139,7 +159,7 @@ class HalfAPI:
def application(self):
return self.__application
def routes(self):
def halfapi_routes(self):
""" Halfapi default routes
"""

5
halfapi/lib/domain.py
View File

@ -56,9 +56,8 @@ def route_decorator(fct: FunctionType, ret_type: str = 'json') -> Coroutine:
fct_args['halfapi'] = {
'user': request.user if
'user' in request else None,
'config': request.scope['config'],
'domain': request.scope['domain'],
'config': request.scope.get('config', {}),
'domain': request.scope.get('domain', 'unknown'),
}

3
halfapi/lib/domain_middleware.py
View File

@ -32,8 +32,7 @@ class DomainMiddleware(BaseHTTPMiddleware):
"""
request.scope['domain'] = self.domain
request.scope['config'] = self.config['domain_config'][self.domain] \
if self.domain in self.config.get('domain_config', {}) else {}
request.scope['config'] = self.config.copy()
response = await call_next(request)

13
halfapi/lib/routes.py
View File

@ -19,11 +19,11 @@ from types import ModuleType, FunctionType
import yaml
from .domain import gen_router_routes, domain_acls, route_decorator
from .domain import gen_router_routes, domain_acls, route_decorator, domain_schema_dict
from .responses import ORJSONResponse
from .acl import args_check
from ..half_route import HalfRoute
from ..conf import DOMAINSDICT
from . import acl
from ..logging import logger
@ -58,6 +58,11 @@ def gen_domain_routes(m_domain: ModuleType):
Returns:
Generator(HalfRoute)
"""
yield HalfRoute(f'/',
JSONRoute(domain_schema_dict(m_domain)),
[{'acl': acl.public}],
'GET'
)
for path, method, m_router, fct, params in gen_router_routes(m_domain, []):
yield HalfRoute(f'/{path}', fct, params, method)
@ -144,9 +149,11 @@ def api_routes(m_dom: ModuleType) -> Tuple[Dict, Dict]:
def api_acls(request):
""" Returns the list of possible ACLs
# TODO: Rewrite
"""
res = {}
domains = DOMAINSDICT()
domains = {}
doc = 'doc' in request.query_params
for domain, m_domain in domains.items():
res[domain] = {}

3
setup.py
View File

@ -50,7 +50,8 @@ setup(
"orjson>=3.4.7,<4",
"pyyaml>=5.3.1,<6",
"timing-asgi>=0.2.1,<1",
"schema>=0.7.4,<1"
"schema>=0.7.4,<1",
"toml>=0.7.1,<0.8"
],
classifiers=[
"Development Status :: 3 - Alpha",

2
tests/cli/test_cli_run.py
View File

@ -12,7 +12,7 @@ def test_run_noproject(cli_runner):
result = cli_runner.invoke(cli, ['run'])
try:
assert result.exit_code == 1
assert result.exit_code == 0
except AssertionError as exc:
print(result.stdout)
raise exc

21
tests/conftest.py
View File

@ -39,7 +39,10 @@ from halfapi.lib.jwt_middleware import (
def dummy_domain():
yield {
'name': 'dummy_domain',
'router': 'dummy_domain.routers'
'router': 'dummy_domain.routers',
'config': {
'test': True
}
}
@pytest.fixture
@ -253,10 +256,10 @@ def dummy_project():
f'secret = {halfapi_secret}\n',
'port = 3050\n',
'loglevel = debug\n',
'[domain]\n',
'[domain.dummy_domain]\n',
f'name = {domain}\n',
'router = dummy_domain.routers\n',
f'[{domain}]\n',
f'[domain.dummy_domain.config]\n',
'test = True'
])
@ -270,9 +273,11 @@ def application_debug(project_runner):
halfAPI = HalfAPI({
'secret':'turlututu',
'production':False,
'domain': {
'domain': {
'name': 'test_domain',
'router': 'test_domain.routers'
}
},
'config':{
'domain_config': {'test_domain': {'test': True}}
@ -288,9 +293,13 @@ def application_domain(dummy_domain):
return HalfAPI({
'secret':'turlututu',
'production':True,
'domain': dummy_domain,
'config':{
'domain_config': {'dummy_domain': {'test': True}}
'domain': {
'domain': {
**dummy_domain,
'config': {
'test': True
}
}
}
}).application

7
tests/test_app.py
View File

@ -8,6 +8,7 @@ from halfapi.lib.domain import NoDomainsException
def test_halfapi_dummy_domain(dummy_domain):
with patch('starlette.applications.Starlette') as mock:
mock.return_value = MagicMock()
halfapi = HalfAPI({
'domain': dummy_domain
})
config = {}
config['domain'] = {}
config['domain'][dummy_domain['name']] = dummy_domain
halfapi = HalfAPI(config)

10
tests/test_conf.py
View File

@ -7,8 +7,10 @@ class TestConf(TestCase):
def setUp(self):
self.args = {
'domain': {
'dummy_domain': {
'name': 'dummy_domain',
'router': 'dummy_domain.routers'
'router': '.routers'
}
}
}
def tearDown(self):
@ -39,7 +41,6 @@ class TestConf(TestCase):
CONFIG,
SCHEMA,
SECRET,
DOMAINSDICT,
PROJECT_NAME,
HOST,
PORT,
@ -49,9 +50,8 @@ class TestConf(TestCase):
assert isinstance(CONFIG, dict)
assert isinstance(SCHEMA, dict)
assert isinstance(SECRET, str)
assert isinstance(DOMAINSDICT(), dict)
assert isinstance(PROJECT_NAME, str)
assert isinstance(HOST, str)
assert isinstance(PORT, str)
assert str(int(PORT)) == PORT
assert isinstance(PORT, int)
assert int(str(int(PORT))) == PORT
assert isinstance(CONF_DIR, str)

1
tests/test_halfapi.py
View File

@ -2,6 +2,5 @@ from halfapi.halfapi import HalfAPI
def test_methods():
assert 'application' in dir(HalfAPI)
assert 'routes' in dir(HalfAPI)
assert 'version' in dir(HalfAPI)
assert 'version_async' in dir(HalfAPI)