From dbca2f28fbe2144ebbf71176932ad6e5d75e8e45 Mon Sep 17 00:00:00 2001 From: Maxime Alves LIRMM Date: Fri, 3 Dec 2021 17:25:57 +0100 Subject: [PATCH] [conf] use of toml for halfapi configs. re-enable possibility of multiple domains --- Pipfile | 2 + halfapi/cli/domain.py | 16 ++--- halfapi/cli/run.py | 19 +++++- halfapi/conf.py | 105 ++++++++++++++++--------------- halfapi/half_domain.py | 57 +++++++++++++++++ halfapi/halfapi.py | 40 +++++++++--- halfapi/lib/domain.py | 5 +- halfapi/lib/domain_middleware.py | 3 +- halfapi/lib/routes.py | 13 +++- setup.py | 3 +- tests/cli/test_cli_run.py | 2 +- tests/conftest.py | 25 +++++--- tests/test_app.py | 7 ++- tests/test_conf.py | 14 ++--- tests/test_halfapi.py | 1 - 15 files changed, 210 insertions(+), 102 deletions(-) create mode 100644 halfapi/half_domain.py diff --git a/Pipfile b/Pipfile index b5e9c5e..17fe9ee 100644 --- a/Pipfile +++ b/Pipfile @@ -21,6 +21,8 @@ pyjwt = ">=2.3.0,<2.4.0" pyyaml = ">=5.3.1,<6" timing-asgi = ">=0.2.1,<1" schema = ">=0.7.4,<1" +toml = "*" +pip = "*" [scripts] halfapi = "python -m halfapi" diff --git a/halfapi/cli/domain.py b/halfapi/cli/domain.py index fb1835b..832d468 100644 --- a/halfapi/cli/domain.py +++ b/halfapi/cli/domain.py @@ -13,7 +13,7 @@ import orjson from .cli import cli -from ..conf import config, write_config, DOMAINSDICT +from ..conf import write_config from ..lib.domain import domain_schema from ..lib.schemas import schema_dict_dom @@ -61,13 +61,7 @@ def create_domain(domain_name: str, module_path: str): os.mkdir(router_path) create_init(router_path) - - if not config.has_section('domain'): - config.add_section('domain') - - config.set('domain', 'name', domain_name) - config.set('domain', 'router', module_path) - write_config() + # TODO: Generate config file domain_tree_create() """ @@ -113,11 +107,13 @@ def list_routes(domain, m_dom): def list_api_routes(): """ Echoes the list of all active domains. + + TODO: Rewrite function """ click.echo('# API Routes') - for domain, m_dom in DOMAINSDICT().items(): - list_routes(domain, m_dom) + # for domain, m_dom in DOMAINSDICT().items(): + # list_routes(domain, m_dom) @click.option('--read',default=True, is_flag=True) diff --git a/halfapi/cli/run.py b/halfapi/cli/run.py index c43cec7..5066806 100644 --- a/halfapi/cli/run.py +++ b/halfapi/cli/run.py @@ -10,9 +10,10 @@ import uvicorn from .cli import cli from .domain import list_api_routes from ..conf import (PROJECT_NAME, HOST, PORT, SCHEMA, - PRODUCTION, LOGLEVEL, DOMAINSDICT, CONFIG, DOMAIN, ROUTER) + PRODUCTION, LOGLEVEL, CONFIG) from ..logging import logger from ..lib.schemas import schema_csv_dict +from ..half_domain import HalfDomain @click.option('--host', default=HOST) @click.option('--port', default=PORT) @@ -26,7 +27,8 @@ from ..lib.schemas import schema_csv_dict @click.argument('schema', type=click.File('r'), required=False) @click.argument('domain', required=False) @cli.command() -def run(host, port, reload, secret, production, loglevel, prefix, check, dryrun, schema, domain): +def run(host, port, reload, secret, production, loglevel, prefix, check, dryrun, + schema, domain): """ The "halfapi run" command """ @@ -58,6 +60,19 @@ def run(host, port, reload, secret, production, loglevel, prefix, check, dryrun, for key, val in schema_csv_dict(schema, prefix).items(): SCHEMA[key] = val + if domain: + # If we specify a domain to run as argument + + for key in CONFIG['domain']: + # Disable all domains + CONFIG['domain'].pop(key) + + # And activate the desired one, mounted without prefix + CONFIG['domain'][domain] = { + 'name': domain, + 'prefix': False + } + # list_api_routes() click.echo(f'uvicorn.run("halfapi.app:application"\n' \ diff --git a/halfapi/conf.py b/halfapi/conf.py index 647f9c6..6379253 100644 --- a/halfapi/conf.py +++ b/halfapi/conf.py @@ -10,7 +10,6 @@ It uses the following environment variables : It defines the following globals : - PROJECT_NAME (str) - HALFAPI_PROJECT_NAME - - DOMAINSDICT ({domain_name: domain_module}) - HALFAPI_DOMAIN_NAME / HALFAPI_DOMAIN_MODULE - PRODUCTION (bool) - HALFAPI_PRODUCTION - LOGLEVEL (string) - HALFAPI_LOGLEVEL - BASE_DIR (str) - HALFAPI_BASE_DIR @@ -18,7 +17,6 @@ It defines the following globals : - PORT (int) - HALFAPI_PORT - CONF_DIR (str) - HALFAPI_CONF_DIR - DRYRUN (bool) - HALFAPI_DRYRUN - - config (ConfigParser) It reads the following ressource : @@ -30,43 +28,41 @@ It follows the following format : name = PROJECT_NAME halfapi_version = HALFAPI_VERSION - [domains] - domain_name = requirements-like-url + [domain.domain_name] + name = domain_name + routers = routers + + [domain.domain_name.config] + option = Argh + """ import logging import os from os import environ import sys -from configparser import ConfigParser import importlib +import tempfile +import uuid + +import toml from .lib.domain import d_domains from .logging import logger +CONFIG = {} -PROJECT_NAME = environ.get('HALFAPI_PROJECT_NAME') or os.path.basename(os.getcwd()) -DOMAINSDICT = lambda: {} -DOMAINS = {} PRODUCTION = True LOGLEVEL = 'info' -HOST = '127.0.0.1' -PORT = '3000' -SECRET = '' CONF_FILE = os.environ.get('HALFAPI_CONF_FILE', '.halfapi/config') DRYRUN = bool(os.environ.get('HALFAPI_DRYRUN', False)) -DOMAIN = None -ROUTER = None SCHEMA = {} -config = ConfigParser(allow_no_value=True) - CONF_DIR = environ.get('HALFAPI_CONF_DIR', '/etc/half_api') HALFAPI_ETC_FILE=os.path.join( - CONF_DIR, 'default.ini' + CONF_DIR, 'config' ) - HALFAPI_DOT_FILE=os.path.join( os.getcwd(), '.halfapi', 'config') @@ -85,59 +81,65 @@ def write_config(): """ Writes the current config to the highest priority config file """ - with open(conf_files()[-1], 'w') as halfapi_config: - config.write(halfapi_config) + # with open(conf_files()[-1], 'w') as halfapi_config: + # config.write(halfapi_config) + pass -def config_dict(): - """ - The config object as a dict - """ - return { - section: dict(config.items(section)) - for section in config.sections() - } def read_config(): """ The highest index in "filenames" are the highest priorty """ - config.read(HALFAPI_CONFIG_FILES) - - + return toml.load(HALFAPI_CONFIG_FILES) CONFIG = {} -read_config() -PROJECT_NAME = config.get('project', 'name', fallback=PROJECT_NAME) +PROJECT_NAME = CONFIG.get('project', {}).get( + 'name', + environ.get('HALFAPI_PROJECT_NAME', os.path.basename(os.getcwd()))) -if len(PROJECT_NAME) == 0: - raise Exception('Need a project name as argument') +if len(CONFIG.get('domain', {}).keys()) == 0: + logger.info('No domains') + # logger.info('Running without domains: %s', d_domains(config) or 'empty domain dictionary') -DOMAINSDICT = lambda: d_domains(config) -DOMAINS = DOMAINSDICT() -if len(DOMAINS) == 0: - logger.info('Running without domains: %s', d_domains(config) or 'empty domain dictionary') -HOST = config.get('project', 'host', fallback=environ.get('HALFAPI_HOST', '127.0.0.1')) -PORT = config.getint('project', 'port', fallback=environ.get('HALFAPI_PORT', '3000')) +# Bind +HOST = CONFIG.get('project', {}).get( + 'host', + environ.get('HALFAPI_HOST', '127.0.0.1')) +PORT = int(CONFIG.get('project', {}).get( + 'port', + environ.get('HALFAPI_PORT', '3000'))) + + +# Secret +SECRET = CONFIG.get('project', {}).get( + 'secret', + environ.get('HALFAPI_SECRET')) + +if not SECRET: + # TODO: Create a temporary secret + _, SECRET = tempfile.mkstemp() + with open('SECRET', 'w') as secret_file: + secret_file.write(str(uuid.uuid4())) -secret_path = config.get('project', 'secret', fallback=environ.get('HALFAPI_SECRET', '')) try: - with open(secret_path, 'r') as secret_file: - - SECRET = secret_file.read().strip() + with open(SECRET, 'r') as secret_file: CONFIG['secret'] = SECRET.strip() except FileNotFoundError as exc: - logger.info('Running without secret file: %s', secret_path or 'no file specified') + logger.info('Running without secret file: %s', SECRET or 'no file specified') -PRODUCTION = config.getboolean('project', 'production', - fallback=environ.get('HALFAPI_PROD', True)) +PRODUCTION = bool(CONFIG.get('project', {}).get( + 'production', + environ.get('HALFAPI_PROD', True))) -LOGLEVEL = config.get('project', 'loglevel', - fallback=environ.get('HALFAPI_LOGLEVEL', 'info')).lower() +LOGLEVEL = CONFIG.get('project', {}).get( + 'loglevel', + environ.get('HALFAPI_LOGLEVEL', 'info')).lower() -BASE_DIR = config.get('project', 'base_dir', - fallback=environ.get('HALFAPI_BASE_DIR', '.')) +BASE_DIR = CONFIG.get('project', {}).get( + 'base_dir', + environ.get('HALFAPI_BASE_DIR', '.')) CONFIG = { 'project_name': PROJECT_NAME, @@ -146,4 +148,5 @@ CONFIG = { 'host': HOST, 'port': PORT, 'dryrun': DRYRUN, + 'domain': {} } diff --git a/halfapi/half_domain.py b/halfapi/half_domain.py new file mode 100644 index 0000000..16394d2 --- /dev/null +++ b/halfapi/half_domain.py @@ -0,0 +1,57 @@ +import importlib + +from starlette.applications import Starlette +from starlette.routing import Router + +from .half_route import HalfRoute +from .lib.routes import gen_domain_routes, gen_schema_routes +from .lib.domain_middleware import DomainMiddleware +from .logging import logger + +class HalfDomain(Starlette): + def __init__(self, app, domain, router=None, config={}): + self.app = app + + self.m_domain = importlib.import_module(domain) + self.name = getattr('__name__', domain, domain) + + if not router: + self.router = getattr('__router__', domain, '.routers') + else: + self.router = router + + self.m_router = importlib.import_module(self.router, domain) + + self.m_acl = importlib.import_module(f'{domain}.acl') + + self.config = config + + """ + if domain: + m_domain = importlib.import_module(domain) + if not router: + router = getattr('__router__', domain, '.routers') + m_domain_router = importlib.import_module(router, domain) + m_domain_acl = importlib.import_module(f'{domain}.acl') + + if not(m_domain and m_domain_router and m_domain_acl): + raise Exception('Cannot import domain') + + self.schema = domain_schema(m_domain) + + routes = [ Route('/', JSONRoute(self.schema)) ] + """ + + logger.info('HalfDomain creation %s %s', domain, config) + super().__init__( + routes=gen_domain_routes(self.m_router), + middleware=[ + (DomainMiddleware, + { + 'domain': self.name, + 'config': self.config + } + ) + ] + ) + diff --git a/halfapi/halfapi.py b/halfapi/halfapi.py index a588e6a..b16d2a0 100644 --- a/halfapi/halfapi.py +++ b/halfapi/halfapi.py @@ -40,13 +40,14 @@ from .lib.domain import domain_schema_dict, NoDomainsException, domain_schema from .lib.routes import gen_domain_routes, gen_schema_routes, JSONRoute from .lib.schemas import schema_json, get_acls from .logging import logger, config_logging +from .half_domain import HalfDomain from halfapi import __version__ class HalfAPI: def __init__(self, config, - routes_dict=None): + d_routes=None): config_logging(logging.DEBUG) SECRET = config.get('secret') @@ -55,27 +56,28 @@ class HalfAPI: DRYRUN = config.get('dryrun', False) self.PRODUCTION = PRODUCTION - self.CONFIG = CONFIG + self.CONFIG = config self.SECRET = SECRET self.__application = None + # Domains + """ HalfAPI routes (if not PRODUCTION, includes debug routes) """ routes = [] routes.append( - Route('/', JSONRoute({})) + Mount('/halfapi', routes=list(self.halfapi_routes())) ) - routes.append( - Mount('/halfapi', routes=list(self.routes())) - ) + logger.info('Config: %s', config) + logger.info('Active domains: %s', config.get('domain', {})) - if routes_dict: - # Mount the routes from the routes_dict argument - domain-less mode + if d_routes: + # Mount the routes from the d_routes argument - domain-less mode logger.info('Domain-less mode : the given schema defines the activated routes') - for route in gen_schema_routes(routes_dict): + for route in gen_schema_routes(d_routes): routes.append(route) else: """ @@ -104,6 +106,24 @@ class HalfAPI: on_startup=startup_fcts ) + for key, domain in config.get('domain', {}).items(): + dom_name = domain.get('name', key) + if domain.get('prefix', False): + path = f'/{dom_name}' + else: + path = '/' + + self.__application.mount(path, + Mount('/', + HalfDomain( + self.application, + domain.get('name', key), + domain.get('router'), + config=domain.get('config', {}) + ) + ) + ) + """ self.__application.add_middleware( DomainMiddleware, @@ -139,7 +159,7 @@ class HalfAPI: def application(self): return self.__application - def routes(self): + def halfapi_routes(self): """ Halfapi default routes """ diff --git a/halfapi/lib/domain.py b/halfapi/lib/domain.py index 2587ab6..01ab6d1 100644 --- a/halfapi/lib/domain.py +++ b/halfapi/lib/domain.py @@ -56,9 +56,8 @@ def route_decorator(fct: FunctionType, ret_type: str = 'json') -> Coroutine: fct_args['halfapi'] = { 'user': request.user if 'user' in request else None, - 'config': request.scope['config'], - 'domain': request.scope['domain'], - + 'config': request.scope.get('config', {}), + 'domain': request.scope.get('domain', 'unknown'), } diff --git a/halfapi/lib/domain_middleware.py b/halfapi/lib/domain_middleware.py index 56be847..b1478bd 100644 --- a/halfapi/lib/domain_middleware.py +++ b/halfapi/lib/domain_middleware.py @@ -32,8 +32,7 @@ class DomainMiddleware(BaseHTTPMiddleware): """ request.scope['domain'] = self.domain - request.scope['config'] = self.config['domain_config'][self.domain] \ - if self.domain in self.config.get('domain_config', {}) else {} + request.scope['config'] = self.config.copy() response = await call_next(request) diff --git a/halfapi/lib/routes.py b/halfapi/lib/routes.py index 2fc8b16..58b89a9 100644 --- a/halfapi/lib/routes.py +++ b/halfapi/lib/routes.py @@ -19,11 +19,11 @@ from types import ModuleType, FunctionType import yaml -from .domain import gen_router_routes, domain_acls, route_decorator +from .domain import gen_router_routes, domain_acls, route_decorator, domain_schema_dict from .responses import ORJSONResponse from .acl import args_check from ..half_route import HalfRoute -from ..conf import DOMAINSDICT +from . import acl from ..logging import logger @@ -58,6 +58,11 @@ def gen_domain_routes(m_domain: ModuleType): Returns: Generator(HalfRoute) """ + yield HalfRoute(f'/', + JSONRoute(domain_schema_dict(m_domain)), + [{'acl': acl.public}], + 'GET' + ) for path, method, m_router, fct, params in gen_router_routes(m_domain, []): yield HalfRoute(f'/{path}', fct, params, method) @@ -144,9 +149,11 @@ def api_routes(m_dom: ModuleType) -> Tuple[Dict, Dict]: def api_acls(request): """ Returns the list of possible ACLs + + # TODO: Rewrite """ res = {} - domains = DOMAINSDICT() + domains = {} doc = 'doc' in request.query_params for domain, m_domain in domains.items(): res[domain] = {} diff --git a/setup.py b/setup.py index b71ea57..a3e4aff 100755 --- a/setup.py +++ b/setup.py @@ -50,7 +50,8 @@ setup( "orjson>=3.4.7,<4", "pyyaml>=5.3.1,<6", "timing-asgi>=0.2.1,<1", - "schema>=0.7.4,<1" + "schema>=0.7.4,<1", + "toml>=0.7.1,<0.8" ], classifiers=[ "Development Status :: 3 - Alpha", diff --git a/tests/cli/test_cli_run.py b/tests/cli/test_cli_run.py index 6c7269f..378a728 100644 --- a/tests/cli/test_cli_run.py +++ b/tests/cli/test_cli_run.py @@ -12,7 +12,7 @@ def test_run_noproject(cli_runner): result = cli_runner.invoke(cli, ['run']) try: - assert result.exit_code == 1 + assert result.exit_code == 0 except AssertionError as exc: print(result.stdout) raise exc diff --git a/tests/conftest.py b/tests/conftest.py index 74098ad..0f89d1b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,7 +39,10 @@ from halfapi.lib.jwt_middleware import ( def dummy_domain(): yield { 'name': 'dummy_domain', - 'router': 'dummy_domain.routers' + 'router': 'dummy_domain.routers', + 'config': { + 'test': True + } } @pytest.fixture @@ -253,10 +256,10 @@ def dummy_project(): f'secret = {halfapi_secret}\n', 'port = 3050\n', 'loglevel = debug\n', - '[domain]\n', + '[domain.dummy_domain]\n', f'name = {domain}\n', 'router = dummy_domain.routers\n', - f'[{domain}]\n', + f'[domain.dummy_domain.config]\n', 'test = True' ]) @@ -271,8 +274,10 @@ def application_debug(project_runner): 'secret':'turlututu', 'production':False, 'domain': { - 'name': 'test_domain', - 'router': 'test_domain.routers' + 'domain': { + 'name': 'test_domain', + 'router': 'test_domain.routers' + } }, 'config':{ 'domain_config': {'test_domain': {'test': True}} @@ -288,9 +293,13 @@ def application_domain(dummy_domain): return HalfAPI({ 'secret':'turlututu', 'production':True, - 'domain': dummy_domain, - 'config':{ - 'domain_config': {'dummy_domain': {'test': True}} + 'domain': { + 'domain': { + **dummy_domain, + 'config': { + 'test': True + } + } } }).application diff --git a/tests/test_app.py b/tests/test_app.py index d2eabeb..5c3ed96 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -8,6 +8,7 @@ from halfapi.lib.domain import NoDomainsException def test_halfapi_dummy_domain(dummy_domain): with patch('starlette.applications.Starlette') as mock: mock.return_value = MagicMock() - halfapi = HalfAPI({ - 'domain': dummy_domain - }) + config = {} + config['domain'] = {} + config['domain'][dummy_domain['name']] = dummy_domain + halfapi = HalfAPI(config) diff --git a/tests/test_conf.py b/tests/test_conf.py index f68087d..5984a2f 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -5,10 +5,12 @@ from halfapi.halfapi import HalfAPI class TestConf(TestCase): def setUp(self): - self.args = { + self.args = { 'domain': { - 'name': 'dummy_domain', - 'router': 'dummy_domain.routers' + 'dummy_domain': { + 'name': 'dummy_domain', + 'router': '.routers' + } } } def tearDown(self): @@ -39,7 +41,6 @@ class TestConf(TestCase): CONFIG, SCHEMA, SECRET, - DOMAINSDICT, PROJECT_NAME, HOST, PORT, @@ -49,9 +50,8 @@ class TestConf(TestCase): assert isinstance(CONFIG, dict) assert isinstance(SCHEMA, dict) assert isinstance(SECRET, str) - assert isinstance(DOMAINSDICT(), dict) assert isinstance(PROJECT_NAME, str) assert isinstance(HOST, str) - assert isinstance(PORT, str) - assert str(int(PORT)) == PORT + assert isinstance(PORT, int) + assert int(str(int(PORT))) == PORT assert isinstance(CONF_DIR, str) diff --git a/tests/test_halfapi.py b/tests/test_halfapi.py index adea751..eb6ca97 100644 --- a/tests/test_halfapi.py +++ b/tests/test_halfapi.py @@ -2,6 +2,5 @@ from halfapi.halfapi import HalfAPI def test_methods(): assert 'application' in dir(HalfAPI) - assert 'routes' in dir(HalfAPI) assert 'version' in dir(HalfAPI) assert 'version_async' in dir(HalfAPI)