Skip to content

Database

aloha.db package - Database and middleware connection helpers.

Sync modules (blocking): from aloha.db import PostgresOperator, MySqlOperator, RedisOperator, ... from aloha.db.postgres import PostgresOperator from aloha.db.mysql import MySqlOperator from aloha.db.redis import RedisOperator from aloha.db.mongo import MongoOperator from aloha.db.elasticsearch import ElasticSearchOperator from aloha.db.kafka import KafkaOperator from aloha.db.sqlite import SqliteOperator from aloha.db.duckdb import DuckOperator from aloha.db.oracle import OracledbOperator

Async modules (non-blocking): from aloha.db import PostgresOperator as PostgresOperatorAio, ... from aloha.db.postgres_aio import PostgresOperator from aloha.db.mysql_aio import MySqlOperator from aloha.db.redis_aio import RedisOperator from aloha.db.mongo_aio import MongoOperator from aloha.db.elasticsearch_aio import ElasticSearchOperator from aloha.db.kafka_aio import KafkaOperator from aloha.db.sqlite_aio import SqliteOperator from aloha.db.duckdb_aio import DuckOperator from aloha.db.oracle_aio import OracledbOperator

Base utilities: from aloha.db.base import PasswordVault from aloha.db.base_aio import PasswordVault # async version

Usage example (sync): from aloha.db.postgres import PostgresOperator

op = PostgresOperator(db_config)
result = op.execute_query("SELECT * FROM users")
for row in result:
    print(row)

Usage example (async): from aloha.db.postgres_aio import PostgresOperator

async def main():
    op = PostgresOperator(db_config)
    result = await op.execute_query("SELECT * FROM users")
    async for row in op.execute_query_scalars("SELECT * FROM users"):
        print(row)
    await op.close()

import asyncio
asyncio.run(main())

from .base import PasswordVault

try: from .postgres import PostgresOperator except (ImportError, ModuleNotFoundError): pass

try: from .mysql import MySqlOperator except (ImportError, ModuleNotFoundError): pass

try: from .redis import RedisOperator except (ImportError, ModuleNotFoundError): pass

try: from .mongo import MongoOperator except (ImportError, ModuleNotFoundError): pass

try: from .elasticsearch import ElasticSearchOperator except (ImportError, ModuleNotFoundError): pass

try: from .kafka import ConsumedMessage, KafkaOperator except (ImportError, ModuleNotFoundError): pass

try: from .sqlite import SqliteOperator except (ImportError, ModuleNotFoundError): pass

try: from .duckdb import DuckOperator except (ImportError, ModuleNotFoundError): pass

try: from .oracle import OracledbOperator except (ImportError, ModuleNotFoundError): pass

Async modules (importable as aliases for easy switching)

from .base_aio import PasswordVault as PasswordVaultAio

try: from .postgres_aio import PostgresOperator as PostgresOperatorAio except (ImportError, ModuleNotFoundError): pass

try: from .mysql_aio import MySqlOperator as MySqlOperatorAio except (ImportError, ModuleNotFoundError): pass

try: from .redis_aio import RedisOperator as RedisOperatorAio except (ImportError, ModuleNotFoundError): pass

try: from .mongo_aio import MongoOperator as MongoOperatorAio except (ImportError, ModuleNotFoundError): pass

try: from .elasticsearch_aio import ElasticSearchOperator as ElasticSearchOperatorAio except (ImportError, ModuleNotFoundError): pass

try: from .kafka_aio import ConsumedMessage as ConsumedMessageAio from .kafka_aio import KafkaOperator as KafkaOperatorAio except (ImportError, ModuleNotFoundError): pass

try: from .sqlite_aio import SqliteOperator as SqliteOperatorAio except (ImportError, ModuleNotFoundError): pass

try: from .duckdb_aio import DuckOperator as DuckOperatorAio except (ImportError, ModuleNotFoundError): pass

try: from .oracle_aio import OracledbOperator as OracledbOperatorAio except (ImportError, ModuleNotFoundError): pass

all = ( "ConsumedMessage", "ConsumedMessageAio", "DuckOperator", "DuckOperatorAio", "ElasticSearchOperator", "ElasticSearchOperatorAio", "KafkaOperator", "KafkaOperatorAio", "MongoOperator", "MongoOperatorAio", "MySqlOperator", "MySqlOperatorAio", "OracledbOperator", "OracledbOperatorAio", "PasswordVault", "PasswordVaultAio", # Sync operators "PostgresOperator", # Async operators (aliased) "PostgresOperatorAio", "RedisOperator", "RedisOperatorAio", "SqliteOperator", "SqliteOperatorAio", )

PasswordVault

Password vault manager that provides access to password vault implementations.

Caches vault instances for performance.

Source code in aloha/db/base.py
class PasswordVault:
    """
    Password vault manager that provides access to password vault implementations.

    Caches vault instances for performance.
    """

    _dict_cache_vault: ClassVar[dict] = {}

    @staticmethod
    def get_vault(vault_type: str | None = None, vault_config: dict | None = None, **kwargs) -> vault.BaseVault:
        """
        Get a password vault instance.

        Supports multiple vault types:
        - 'plain' or 'aes': AES-based vault (default fallback)
        - 'cyberark': CyberArk vault
        - Other/None: Dummy vault (plain text)

        :param vault_type: Type of vault to use (overrides config)
        :param vault_config: Vault configuration dictionary
        :param args: Additional arguments
        :param kwargs: Additional keyword arguments
        :return: Vault instance implementing BaseVault interface
        :raises RuntimeError: If CyberArk vault is requested but config is missing
        """
        encryption_method = vault_type or SETTINGS.config.get("PASSWORD_ENCRYPTION")
        LOG.debug("Using password vault: %s", encryption_method)  # nosemgrep

        cache_key = f"{encryption_method}:{vault_config!s}"
        if cache_key not in PasswordVault._dict_cache_vault:
            if encryption_method in ("plain", "aes") or encryption_method is True:
                v = vault.AesVault(**(vault_config or {}))
            elif encryption_method == "cyberark":
                config_cyberark = vault_config or SETTINGS.config.get("CYBERARK_CONFIG")
                if config_cyberark is None:
                    raise RuntimeError("Missing [CYBERARK_CONFIG] in config!")
                v = vault.CyberArkVault(**config_cyberark)
            else:
                msg = f"Using plain password vault as unknown value of PASSWORD_ENCRYPTION={encryption_method} in config."
                LOG.info(msg)  # nosemgrep
                v = vault.DummyVault(**(vault_config or {}))
            PasswordVault._dict_cache_vault[cache_key] = v

        return PasswordVault._dict_cache_vault[cache_key]

get_vault(vault_type=None, vault_config=None, **kwargs) staticmethod

Get a password vault instance.

Supports multiple vault types: - 'plain' or 'aes': AES-based vault (default fallback) - 'cyberark': CyberArk vault - Other/None: Dummy vault (plain text)

Parameters:

Name Type Description Default
vault_type str | None

Type of vault to use (overrides config)

None
vault_config dict | None

Vault configuration dictionary

None
args

Additional arguments

required
kwargs

Additional keyword arguments

{}

Returns:

Type Description
BaseVault

Vault instance implementing BaseVault interface

Raises:

Type Description
RuntimeError

If CyberArk vault is requested but config is missing

Source code in aloha/db/base.py
@staticmethod
def get_vault(vault_type: str | None = None, vault_config: dict | None = None, **kwargs) -> vault.BaseVault:
    """
    Get a password vault instance.

    Supports multiple vault types:
    - 'plain' or 'aes': AES-based vault (default fallback)
    - 'cyberark': CyberArk vault
    - Other/None: Dummy vault (plain text)

    :param vault_type: Type of vault to use (overrides config)
    :param vault_config: Vault configuration dictionary
    :param args: Additional arguments
    :param kwargs: Additional keyword arguments
    :return: Vault instance implementing BaseVault interface
    :raises RuntimeError: If CyberArk vault is requested but config is missing
    """
    encryption_method = vault_type or SETTINGS.config.get("PASSWORD_ENCRYPTION")
    LOG.debug("Using password vault: %s", encryption_method)  # nosemgrep

    cache_key = f"{encryption_method}:{vault_config!s}"
    if cache_key not in PasswordVault._dict_cache_vault:
        if encryption_method in ("plain", "aes") or encryption_method is True:
            v = vault.AesVault(**(vault_config or {}))
        elif encryption_method == "cyberark":
            config_cyberark = vault_config or SETTINGS.config.get("CYBERARK_CONFIG")
            if config_cyberark is None:
                raise RuntimeError("Missing [CYBERARK_CONFIG] in config!")
            v = vault.CyberArkVault(**config_cyberark)
        else:
            msg = f"Using plain password vault as unknown value of PASSWORD_ENCRYPTION={encryption_method} in config."
            LOG.info(msg)  # nosemgrep
            v = vault.DummyVault(**(vault_config or {}))
        PasswordVault._dict_cache_vault[cache_key] = v

    return PasswordVault._dict_cache_vault[cache_key]

main()

Command-line tool to decrypt passwords from config.

Usage: python -m aloha.db.base

Source code in aloha/db/base.py
def main():
    """
    Command-line tool to decrypt passwords from config.

    Usage: python -m aloha.db.base <config_key>
    """
    import sys

    config_key = sys.argv[-1]
    LOG.debug(f"Getting pwd for deploy key [deploy.{config_key}]")
    try:
        db_config = SETTINGS.config["deploy"][config_key]
        password_vault = PasswordVault.get_vault()
        p = password_vault.get_password(db_config.get("password"))
        LOG.debug(f"Decrypted PWD: {p}")
    except KeyError:
        LOG.error(f"Please make sure config key [deploy.{config_key}] exists!")

DuckDB connection helpers.

DuckOperator

Create and use a DuckDB connection through SQLAlchemy.

Source code in aloha/db/duckdb.py
class DuckOperator:
    """Create and use a DuckDB connection through SQLAlchemy."""

    def __init__(self, db_config, **kwargs):
        """Build a DuckDB engine, creating the database file if necessary."""
        """db_config example:
        {
            "path": "/path/to/db.duckdb",     # file path of duckdb, use ":memory:" for in-memory mode
            "schema": "sales",                # optional, 'main' by default
            "read_only": True,                # optional, False by default, (will set to False if in in-memory mode)
            "config": {"memory_limit": "500mb"}, # optional, duckdb connection configs
        }
        """
        self._config = {
            "path": db_config.get("path", ":memory:"),
            "schema": db_config.get("schema", "main"),
            "read_only": bool(db_config.get("read_only", False)),
            "config": db_config.get("config", {}),
            "auto_commit": db_config.get("auto_commit", True),
        }

        if not self._config["path"] or self._config["path"] == ":memory:":  # in-memory mode
            self._config["path"] = ":memory:"

            if self._config["read_only"]:  # in-memory mode cannot be read-only
                LOG.warning("In-memory database cannot be read-only. Setting read_only=False.")
                self._config["read_only"] = False

        else:
            self._prepare_database()

        try:
            str_connection = f"duckdb:///{self._config['path']}"
            self.engine = create_engine(
                str_connection,
                connect_args={"read_only": self._config["read_only"], "config": self._config["config"]},
                **kwargs,
            )

            self._initialize_schema()
            msg = f"DuckDB connected: {self._config['path']} [schema={self._config['schema']}, read_only={self._config['read_only']}]"
            LOG.debug(msg)
        except Exception as e:
            LOG.exception(e)
            raise RuntimeError("Failed to connect to DuckDB") from e

    def _prepare_database(self):
        """Prepare the database file and its parent directory."""
        path = self._config["path"]
        path_obj = Path(path)

        parent_dir = path_obj.parent
        if not parent_dir.exists():
            if self._config["read_only"]:
                raise RuntimeError(f"Directory '{parent_dir}' does not exist and read_only=True")
            try:
                parent_dir.mkdir(parents=True, exist_ok=True)
                LOG.debug(f"Created directory: {parent_dir}")
            except Exception as e:
                raise RuntimeError(f"Failed to create directory '{parent_dir}': {e}") from e

        if not path_obj.exists():
            if self._config["read_only"]:
                raise RuntimeError(f"DuckDB file '{path}' does not exist and read_only=True")
            try:
                LOG.debug(f"Database file not found, creating: {path}")
                duckdb.connect(path).close()
            except Exception as e:
                raise RuntimeError(f"Failed to create database file '{path}': {e}") from e

    def _initialize_schema(self):
        """Create or select the requested schema."""
        if self._config["schema"] == "main":
            return

        try:
            schema = self._config["schema"]
            if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", schema):
                raise ValueError(f"Invalid schema name: {schema!r}")
            schema_quoted = f'"{schema}"'
            if self._config["read_only"]:
                result = self.engine.connext().execute(
                    text("SELECT schema_name FROM information_schema.schemata WHERE schema_name = :schema"),
                    {"schema": schema},
                )
                if not result.fetchone():
                    raise RuntimeError(f"Schema '{schema}' does not exist and read_only=True")
            else:
                self.engine.connect().exec_driver_sql(f"CREATE SCHEMA IF NOT EXISTS {schema_quoted}")

            self.engine.connect().exec_driver_sql(f"SET schema '{schema}'")
        except Exception as e:
            raise RuntimeError(f"Failed to initialize schema: {e}") from e

    @property
    def connection(self):
        return self.engine

    conn = connection

    def execute_query(self, sql, *args, **kwargs):
        """Execute a SQL statement and return the cursor result."""
        with self.engine.connect() as conn:
            cur = conn.execute(text(sql), *args, *kwargs)
            if self._config.get("auto_commit", True):
                conn.commit()
            return cur

    @property
    def connection_str(self) -> str:
        """Return a human-readable connection string."""
        return f"duckdb:///{self._config['path']} [schema={self._config['schema']}, read_only={self._config['read_only']}]"

connection_str property

Return a human-readable connection string.

__init__(db_config, **kwargs)

Build a DuckDB engine, creating the database file if necessary.

Source code in aloha/db/duckdb.py
def __init__(self, db_config, **kwargs):
    """Build a DuckDB engine, creating the database file if necessary."""
    """db_config example:
    {
        "path": "/path/to/db.duckdb",     # file path of duckdb, use ":memory:" for in-memory mode
        "schema": "sales",                # optional, 'main' by default
        "read_only": True,                # optional, False by default, (will set to False if in in-memory mode)
        "config": {"memory_limit": "500mb"}, # optional, duckdb connection configs
    }
    """
    self._config = {
        "path": db_config.get("path", ":memory:"),
        "schema": db_config.get("schema", "main"),
        "read_only": bool(db_config.get("read_only", False)),
        "config": db_config.get("config", {}),
        "auto_commit": db_config.get("auto_commit", True),
    }

    if not self._config["path"] or self._config["path"] == ":memory:":  # in-memory mode
        self._config["path"] = ":memory:"

        if self._config["read_only"]:  # in-memory mode cannot be read-only
            LOG.warning("In-memory database cannot be read-only. Setting read_only=False.")
            self._config["read_only"] = False

    else:
        self._prepare_database()

    try:
        str_connection = f"duckdb:///{self._config['path']}"
        self.engine = create_engine(
            str_connection,
            connect_args={"read_only": self._config["read_only"], "config": self._config["config"]},
            **kwargs,
        )

        self._initialize_schema()
        msg = f"DuckDB connected: {self._config['path']} [schema={self._config['schema']}, read_only={self._config['read_only']}]"
        LOG.debug(msg)
    except Exception as e:
        LOG.exception(e)
        raise RuntimeError("Failed to connect to DuckDB") from e

execute_query(sql, *args, **kwargs)

Execute a SQL statement and return the cursor result.

Source code in aloha/db/duckdb.py
def execute_query(self, sql, *args, **kwargs):
    """Execute a SQL statement and return the cursor result."""
    with self.engine.connect() as conn:
        cur = conn.execute(text(sql), *args, *kwargs)
        if self._config.get("auto_commit", True):
            conn.commit()
        return cur

ElasticSearchOperator

Create and use an Elasticsearch client with optional index helpers.

Source code in aloha/db/elasticsearch.py
class ElasticSearchOperator:
    """Create and use an Elasticsearch client with optional index helpers."""

    def __init__(self, config, index_config=None):
        """Build the client and optionally load the index configuration."""
        self.es_config = config

        password_vault = PasswordVault.get_vault(config.get("vault_type"), config.get("vault_config"))
        username = config.get("username")
        password = password_vault.get_password(config.get("password"))

        hosts = config.get("host", "localhost")
        masked_hosts = _mask_hosts(hosts)
        LOG.debug("ElasticSearch connection info: " + str(masked_hosts))

        self._config = {
            "http_auth": (username, password) if username is not None and password is not None else None,
            "hosts": hosts,
            "timeout": config.get("timeout", 0.1),
            "max_retries": config.get("max_retries", 3),
            "retry_on_timeout": config.get("retry_on_timeout", True),
        }

        self.index_config = index_config
        self.index_name = self.es_config.get("index_name")
        self.index_type = self.es_config.get("index_type")

        self.es = Elasticsearch(**self._config)

        if index_config is not None:
            self.index_config = self._load_config(index_config)

    @staticmethod
    def _load_config(config):
        """Load an index configuration from a dict or JSON file."""
        if isinstance(config, dict):
            return config

        elif isinstance(config, str) and ".json" in config:
            with open(config, "r", encoding="utf-8") as f:
                config = json.load(f)
            return config

        else:
            raise ValueError("Invalid ES config data type")

    def put_mapping(self, index_name=None, index_type=None, index_config: dict | None = None):
        """Apply a mapping definition to the current index."""
        return self.es.indices.put_mapping(
            index=index_name or self.index_name,
            doc_type=index_type or self.index_type,
            body=index_config["mappings"][index_type or self.index_type],
        )

    def build_index(self, index_name=None, index_config=None, raise_if_exist=False):
        """Create the index if it does not already exist."""
        if self.es.indices.exists(index=index_name or self.index_name) is not True:
            res = self.es.indices.create(index=index_name or self.index_name, body=index_config or self.index_config)
            return res
        else:
            msg = f"Index [{self.index_name}] already exits"
            if raise_if_exist:
                raise RuntimeError(msg)
            else:
                LOG.info(msg)
                return False

    def search(self, query, index_name=None, index_type=None):
        """Execute a search query."""
        return self.es.search(index=index_name or self.index_name, doc_type=index_type or self.index_type, body=query)

    def msearch(self, body):
        """Execute a multi-search request."""
        return self.es.msearch(body=body)

    def insert(self, doc, index_name=None, index_type=None, id=None):
        """Insert or replace a document."""
        return self.es.index(index=index_name or self.index_name, doc_type=index_type or self.index_type, id=id, body=doc)

__init__(config, index_config=None)

Build the client and optionally load the index configuration.

Source code in aloha/db/elasticsearch.py
def __init__(self, config, index_config=None):
    """Build the client and optionally load the index configuration."""
    self.es_config = config

    password_vault = PasswordVault.get_vault(config.get("vault_type"), config.get("vault_config"))
    username = config.get("username")
    password = password_vault.get_password(config.get("password"))

    hosts = config.get("host", "localhost")
    masked_hosts = _mask_hosts(hosts)
    LOG.debug("ElasticSearch connection info: " + str(masked_hosts))

    self._config = {
        "http_auth": (username, password) if username is not None and password is not None else None,
        "hosts": hosts,
        "timeout": config.get("timeout", 0.1),
        "max_retries": config.get("max_retries", 3),
        "retry_on_timeout": config.get("retry_on_timeout", True),
    }

    self.index_config = index_config
    self.index_name = self.es_config.get("index_name")
    self.index_type = self.es_config.get("index_type")

    self.es = Elasticsearch(**self._config)

    if index_config is not None:
        self.index_config = self._load_config(index_config)

build_index(index_name=None, index_config=None, raise_if_exist=False)

Create the index if it does not already exist.

Source code in aloha/db/elasticsearch.py
def build_index(self, index_name=None, index_config=None, raise_if_exist=False):
    """Create the index if it does not already exist."""
    if self.es.indices.exists(index=index_name or self.index_name) is not True:
        res = self.es.indices.create(index=index_name or self.index_name, body=index_config or self.index_config)
        return res
    else:
        msg = f"Index [{self.index_name}] already exits"
        if raise_if_exist:
            raise RuntimeError(msg)
        else:
            LOG.info(msg)
            return False

insert(doc, index_name=None, index_type=None, id=None)

Insert or replace a document.

Source code in aloha/db/elasticsearch.py
def insert(self, doc, index_name=None, index_type=None, id=None):
    """Insert or replace a document."""
    return self.es.index(index=index_name or self.index_name, doc_type=index_type or self.index_type, id=id, body=doc)

msearch(body)

Execute a multi-search request.

Source code in aloha/db/elasticsearch.py
def msearch(self, body):
    """Execute a multi-search request."""
    return self.es.msearch(body=body)

put_mapping(index_name=None, index_type=None, index_config=None)

Apply a mapping definition to the current index.

Source code in aloha/db/elasticsearch.py
def put_mapping(self, index_name=None, index_type=None, index_config: dict | None = None):
    """Apply a mapping definition to the current index."""
    return self.es.indices.put_mapping(
        index=index_name or self.index_name,
        doc_type=index_type or self.index_type,
        body=index_config["mappings"][index_type or self.index_type],
    )

search(query, index_name=None, index_type=None)

Execute a search query.

Source code in aloha/db/elasticsearch.py
def search(self, query, index_name=None, index_type=None):
    """Execute a search query."""
    return self.es.search(index=index_name or self.index_name, doc_type=index_type or self.index_type, body=query)

Kafka connection helpers.

ConsumedMessage dataclass

Represents a message consumed from Kafka.

Source code in aloha/db/kafka.py
@dataclass
class ConsumedMessage:
    """Represents a message consumed from Kafka."""

    topic: str
    partition: int
    offset: int
    key: str | bytes | None
    value: str | bytes | None
    headers: list[tuple[str, str | bytes]] | None = None

KafkaOperator

Create Kafka admin, producer, and consumer clients.

Source code in aloha/db/kafka.py
class KafkaOperator:
    """Create Kafka admin, producer, and consumer clients."""

    def __init__(self, kafka_config):
        """
        Parameter reference: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md

        :param kafka_config:
        host = [
            {host: kafka_server_1, port: 9092}
        ]
        """
        self._config = json.loads(json.dumps(kafka_config, ensure_ascii=False))  # deep copy

        if "host" in kafka_config:
            self._config = {
                "bootstrap.servers": ",".join(["{host}:{port}".format(**i) for i in kafka_config.pop("host")]),
            }
        LOG.debug("Kafka connection info: " + str(self._config))
        self._producer = None
        self._consumer = None

    def admin_client(self, *args, **kwargs):
        """Return a configured Kafka AdminClient."""
        config_admin = {**self._config}
        a = kafka_admin.AdminClient(config_admin)
        return a

    def create_topic(self, topic: str, num_partitions=3, replication_factor=1, *args, **kwargs):
        """Create a Kafka topic and wait for the broker response."""
        """Note: In a multi-cluster production scenario, it is more typical to use a replication_factor of 3 for durability."""
        a = self.admin_client()
        new_topic = kafka_admin.NewTopic(topic, num_partitions=num_partitions, replication_factor=replication_factor)

        # Call create_topics to asynchronously create topics. A dict of <topic,future> is returned.
        fs = a.create_topics([new_topic])

        # Wait for each operation to finish.
        for topic_name, f in fs.items():
            try:
                f.result()  # The result itself is None
                LOG.info(f"Topic {topic_name} created")
            except Exception as e:  # noqa: BLE001
                LOG.error(f"Failed to create topic {topic_name}: {e}")
                return False
            finally:
                if hasattr(a, "close"):
                    a.close()

        return True

    def producer(self) -> kafka.Producer:
        """Return a configured Kafka Producer."""
        if self._producer is None:
            config_producer = {**self._config}
            self._producer = kafka.Producer(config_producer)
        return self._producer

    def producer_deliver(
        self, topic: str, generator: typing.Iterator[typing.Any], func_callback: callable | None = None, *args, **kwargs
    ):
        """Stream messages from an iterator into a Kafka topic."""
        # func_callback should be a function that takes two arguments: err and msg
        p = self.producer()

        def delivery_report(err, msg):
            """Called once for each message produced to indicate delivery result. Triggered by poll() or flush()."""
            if err is not None:
                LOG.error(f"Kafka msg delivery failed: {err}")
            else:
                LOG.debug(f"Kafka msg delivered to {msg.topic()} [{msg.partition()}]")

        if func_callback is None:
            func_callback = delivery_report

        for data in generator:  # some data from the generator
            # Trigger any available delivery report callbacks from previous produce() calls
            p.poll(0)

            value, key, headers = _unpack_message(data)
            if isinstance(value, str):
                value = value.encode("utf-8")
            prepared_headers = _prepare_headers(headers)

            # Asynchronously produce a message, the delivery report callback
            # will be triggered from poll() above, or flush() below, when the message has
            # been successfully delivered or failed permanently.
            p.produce(topic, value=value, key=key, headers=prepared_headers, callback=func_callback)

        # Wait for any outstanding messages to be delivered and delivery report callbacks to be triggered.
        p.flush()

    def consumer_generator(
        self, topics_subscribe: list, group_id: str | None = None, poll_timeout: float = 1.0, *args, **kwargs
    ) -> typing.Iterator[ConsumedMessage]:
        """Yield decoded messages from the subscribed Kafka topics."""
        config_consumer = {"auto.offset.reset": "earliest", "enable.auto.commit": False, **self._config}
        if group_id is not None:
            config_consumer["group.id"] = group_id

        # Merge extra config passed via kwargs (convert snake_case to dot.case for confluent_kafka)
        for k, v in kwargs.items():
            k_dot = k.replace("_", ".")
            config_consumer[k_dot] = v

        c = kafka.Consumer(config_consumer)
        self._consumer = c

        c.subscribe(topics_subscribe)
        try:
            while True:
                msg = c.poll(poll_timeout)

                if msg is None:
                    continue
                elif msg.error():
                    code = msg.error().code()
                    if code == kafka.KafkaError._PARTITION_EOF:
                        pass
                    LOG.error(f"Kafka consumer: {msg.error()}")
                    continue

                val = msg.value()
                if val is not None:
                    try:
                        val = val.decode("utf-8")
                    except UnicodeDecodeError:
                        pass

                key = msg.key()
                if key is not None:
                    try:
                        key = key.decode("utf-8")
                    except UnicodeDecodeError:
                        pass

                raw_headers = msg.headers()
                headers = None
                if raw_headers:
                    headers = []
                    for k, v in raw_headers:
                        if isinstance(v, bytes):
                            try:
                                v = v.decode("utf-8")
                            except UnicodeDecodeError:
                                pass
                        headers.append((k, v))

                consumed_msg = ConsumedMessage(
                    topic=msg.topic(),
                    partition=msg.partition(),
                    offset=msg.offset(),
                    key=key,
                    value=val,
                    headers=headers,
                )
                LOG.debug(f"Received message: {consumed_msg}")
                yield consumed_msg
        finally:
            c.close()
            self._consumer = None

    def commit(self, *args, **kwargs):
        """Commit offsets for the current consumer."""
        if self._consumer is not None:
            self._consumer.commit(*args, **kwargs)

    def close(self):
        """Close all Kafka clients."""
        if self._producer is not None:
            self._producer.flush()
            self._producer = None
        if self._consumer is not None:
            self._consumer.close()
            self._consumer = None

    def __enter__(self):
        """Context manager entry."""
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        """Context manager exit."""
        self.close()

__enter__()

Context manager entry.

Source code in aloha/db/kafka.py
def __enter__(self):
    """Context manager entry."""
    return self

__exit__(exc_type, exc_val, exc_tb)

Context manager exit.

Source code in aloha/db/kafka.py
def __exit__(self, exc_type, exc_val, exc_tb):
    """Context manager exit."""
    self.close()

__init__(kafka_config)

Parameter reference: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md

Parameters:

Name Type Description Default
kafka_config

host = [ {host: kafka_server_1, port: 9092} ]

required
Source code in aloha/db/kafka.py
def __init__(self, kafka_config):
    """
    Parameter reference: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md

    :param kafka_config:
    host = [
        {host: kafka_server_1, port: 9092}
    ]
    """
    self._config = json.loads(json.dumps(kafka_config, ensure_ascii=False))  # deep copy

    if "host" in kafka_config:
        self._config = {
            "bootstrap.servers": ",".join(["{host}:{port}".format(**i) for i in kafka_config.pop("host")]),
        }
    LOG.debug("Kafka connection info: " + str(self._config))
    self._producer = None
    self._consumer = None

admin_client(*args, **kwargs)

Return a configured Kafka AdminClient.

Source code in aloha/db/kafka.py
def admin_client(self, *args, **kwargs):
    """Return a configured Kafka AdminClient."""
    config_admin = {**self._config}
    a = kafka_admin.AdminClient(config_admin)
    return a

close()

Close all Kafka clients.

Source code in aloha/db/kafka.py
def close(self):
    """Close all Kafka clients."""
    if self._producer is not None:
        self._producer.flush()
        self._producer = None
    if self._consumer is not None:
        self._consumer.close()
        self._consumer = None

commit(*args, **kwargs)

Commit offsets for the current consumer.

Source code in aloha/db/kafka.py
def commit(self, *args, **kwargs):
    """Commit offsets for the current consumer."""
    if self._consumer is not None:
        self._consumer.commit(*args, **kwargs)

consumer_generator(topics_subscribe, group_id=None, poll_timeout=1.0, *args, **kwargs)

Yield decoded messages from the subscribed Kafka topics.

Source code in aloha/db/kafka.py
def consumer_generator(
    self, topics_subscribe: list, group_id: str | None = None, poll_timeout: float = 1.0, *args, **kwargs
) -> typing.Iterator[ConsumedMessage]:
    """Yield decoded messages from the subscribed Kafka topics."""
    config_consumer = {"auto.offset.reset": "earliest", "enable.auto.commit": False, **self._config}
    if group_id is not None:
        config_consumer["group.id"] = group_id

    # Merge extra config passed via kwargs (convert snake_case to dot.case for confluent_kafka)
    for k, v in kwargs.items():
        k_dot = k.replace("_", ".")
        config_consumer[k_dot] = v

    c = kafka.Consumer(config_consumer)
    self._consumer = c

    c.subscribe(topics_subscribe)
    try:
        while True:
            msg = c.poll(poll_timeout)

            if msg is None:
                continue
            elif msg.error():
                code = msg.error().code()
                if code == kafka.KafkaError._PARTITION_EOF:
                    pass
                LOG.error(f"Kafka consumer: {msg.error()}")
                continue

            val = msg.value()
            if val is not None:
                try:
                    val = val.decode("utf-8")
                except UnicodeDecodeError:
                    pass

            key = msg.key()
            if key is not None:
                try:
                    key = key.decode("utf-8")
                except UnicodeDecodeError:
                    pass

            raw_headers = msg.headers()
            headers = None
            if raw_headers:
                headers = []
                for k, v in raw_headers:
                    if isinstance(v, bytes):
                        try:
                            v = v.decode("utf-8")
                        except UnicodeDecodeError:
                            pass
                    headers.append((k, v))

            consumed_msg = ConsumedMessage(
                topic=msg.topic(),
                partition=msg.partition(),
                offset=msg.offset(),
                key=key,
                value=val,
                headers=headers,
            )
            LOG.debug(f"Received message: {consumed_msg}")
            yield consumed_msg
    finally:
        c.close()
        self._consumer = None

create_topic(topic, num_partitions=3, replication_factor=1, *args, **kwargs)

Create a Kafka topic and wait for the broker response.

Source code in aloha/db/kafka.py
def create_topic(self, topic: str, num_partitions=3, replication_factor=1, *args, **kwargs):
    """Create a Kafka topic and wait for the broker response."""
    """Note: In a multi-cluster production scenario, it is more typical to use a replication_factor of 3 for durability."""
    a = self.admin_client()
    new_topic = kafka_admin.NewTopic(topic, num_partitions=num_partitions, replication_factor=replication_factor)

    # Call create_topics to asynchronously create topics. A dict of <topic,future> is returned.
    fs = a.create_topics([new_topic])

    # Wait for each operation to finish.
    for topic_name, f in fs.items():
        try:
            f.result()  # The result itself is None
            LOG.info(f"Topic {topic_name} created")
        except Exception as e:  # noqa: BLE001
            LOG.error(f"Failed to create topic {topic_name}: {e}")
            return False
        finally:
            if hasattr(a, "close"):
                a.close()

    return True

producer()

Return a configured Kafka Producer.

Source code in aloha/db/kafka.py
def producer(self) -> kafka.Producer:
    """Return a configured Kafka Producer."""
    if self._producer is None:
        config_producer = {**self._config}
        self._producer = kafka.Producer(config_producer)
    return self._producer

producer_deliver(topic, generator, func_callback=None, *args, **kwargs)

Stream messages from an iterator into a Kafka topic.

Source code in aloha/db/kafka.py
def producer_deliver(
    self, topic: str, generator: typing.Iterator[typing.Any], func_callback: callable | None = None, *args, **kwargs
):
    """Stream messages from an iterator into a Kafka topic."""
    # func_callback should be a function that takes two arguments: err and msg
    p = self.producer()

    def delivery_report(err, msg):
        """Called once for each message produced to indicate delivery result. Triggered by poll() or flush()."""
        if err is not None:
            LOG.error(f"Kafka msg delivery failed: {err}")
        else:
            LOG.debug(f"Kafka msg delivered to {msg.topic()} [{msg.partition()}]")

    if func_callback is None:
        func_callback = delivery_report

    for data in generator:  # some data from the generator
        # Trigger any available delivery report callbacks from previous produce() calls
        p.poll(0)

        value, key, headers = _unpack_message(data)
        if isinstance(value, str):
            value = value.encode("utf-8")
        prepared_headers = _prepare_headers(headers)

        # Asynchronously produce a message, the delivery report callback
        # will be triggered from poll() above, or flush() below, when the message has
        # been successfully delivered or failed permanently.
        p.produce(topic, value=value, key=key, headers=prepared_headers, callback=func_callback)

    # Wait for any outstanding messages to be delivered and delivery report callbacks to be triggered.
    p.flush()

MongoDB connection helpers.

MongoOperator(config)

Return a cached MongoDB operation wrapper for the given config.

Source code in aloha/db/mongo.py
def MongoOperator(config):
    """Return a cached MongoDB operation wrapper for the given config."""
    db_name = config.get("db_name")
    collection_name = config.get("collection_name")

    _config = {k: v for k, v in config.items() if v is not None}
    key = "{}:{}:{}".format(json.dumps(_config, sort_keys=True, ensure_ascii=False), db_name or "", collection_name or "")

    if key not in _conn:
        try:
            _conn[key] = _MongoDBOperation(_config, db_name=db_name, collection_name=collection_name)
        except Exception as e:  # noqa: BLE001
            LOG.exception(e)
            return
    return _conn[key]

MySQL connection helpers.

MySqlOperator

Create and use a SQLAlchemy-backed MySQL connection.

Source code in aloha/db/mysql.py
class MySqlOperator:
    """Create and use a SQLAlchemy-backed MySQL connection."""

    def __init__(self, db_config, **kwargs):
        """Build a connection pool from the provided database config."""
        password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config"))
        self._config = {
            "host": db_config["host"],
            "port": db_config["port"],
            "user": db_config["user"],
            "password": password_vault.get_password(db_config["password"]),
            "dbname": db_config["dbname"],
        }

        try:
            self.db = create_engine(
                "mysql+pymysql://{user}:{password}@{host}:{port}/{dbname}".format(**self._config),
                pool_size=50,
                pool_recycle=500,
                pool_pre_ping=True,
                **kwargs,
            )
            LOG.debug("MySQL connected: {host}:{port}/{dbname}".format(**self._config))
        except Exception as e:
            LOG.exception(e)
            raise RuntimeError("Failed to connect to MySQL") from e

    @property
    def connection(self):
        return self.db

    def execute_query(self, sql, *args, **kwargs):
        """Execute a SQL statement and return the cursor result."""
        with self.db.connect() as conn:
            cur = conn.execute(text(sql), *args, **kwargs)
            return cur

    @property
    def connection_str(self) -> str:
        """Return a human-readable connection string."""
        return "mysql://{user}:{password}@{host}:{port}/{dbname}".format(**self._config)

connection_str property

Return a human-readable connection string.

__init__(db_config, **kwargs)

Build a connection pool from the provided database config.

Source code in aloha/db/mysql.py
def __init__(self, db_config, **kwargs):
    """Build a connection pool from the provided database config."""
    password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config"))
    self._config = {
        "host": db_config["host"],
        "port": db_config["port"],
        "user": db_config["user"],
        "password": password_vault.get_password(db_config["password"]),
        "dbname": db_config["dbname"],
    }

    try:
        self.db = create_engine(
            "mysql+pymysql://{user}:{password}@{host}:{port}/{dbname}".format(**self._config),
            pool_size=50,
            pool_recycle=500,
            pool_pre_ping=True,
            **kwargs,
        )
        LOG.debug("MySQL connected: {host}:{port}/{dbname}".format(**self._config))
    except Exception as e:
        LOG.exception(e)
        raise RuntimeError("Failed to connect to MySQL") from e

execute_query(sql, *args, **kwargs)

Execute a SQL statement and return the cursor result.

Source code in aloha/db/mysql.py
def execute_query(self, sql, *args, **kwargs):
    """Execute a SQL statement and return the cursor result."""
    with self.db.connect() as conn:
        cur = conn.execute(text(sql), *args, **kwargs)
        return cur

Oracle DB connection helpers.

OracledbOperator

Create and use a SQLAlchemy-backed Oracle connection.

Source code in aloha/db/oracle.py
class OracledbOperator:
    """Create and use a SQLAlchemy-backed Oracle connection."""

    def __init__(self, db_config, **kwargs):
        """Build an Oracle connection pool from the provided config."""
        """example of db_config:
        {
            "host": "192.168.1.100",
            "port": 1521,
            "user": "PT_INDEX",
            "password": "vault_key_or_plain",
            "service_name": "orcl",   # 推荐使用 service_name
            "sid": "orcl",            # 或使用 sid
            "vault_type": "...",
            "vault_config": {...},
            "lib_dir": "/opt/oracle/instantclient"  # optional, use THICK mode if defined.
        }
        """

        password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config"))
        self._config = {
            "host": db_config["host"],
            "port": db_config["port"],
            "user": db_config["user"],
            "password": password_vault.get_password(db_config.get("password")),
        }

        if "lib_dir" in db_config:  # use Thick mode
            try:
                oracledb.init_oracle_client(lib_dir=db_config["lib_dir"])
                LOG.info("Oracle client initialized in THICK mode from: {}".format(db_config["lib_dir"]))
            except Exception as e:
                LOG.warning(f"Warning: {e}")
                raise RuntimeError(f"Failed to initialize Oracle client: {e}") from e

        service_name = db_config.get("service_name")
        sid = db_config.get("sid")

        if service_name:  # using service_name (recommended)
            dsn = oracledb.makedsn(db_config["host"], db_config["port"], service_name=service_name)
        elif sid:  # using SID
            dsn = oracledb.makedsn(db_config["host"], db_config["port"], sid=sid)
        else:
            raise ValueError("Oracle config must specify service_name or sid")

        self._config["dsn"] = dsn
        try:
            self.engine = create_engine(
                "oracle+oracledb://{user}:{password}@".format(**self._config),
                connect_args={"dsn": dsn},
                pool_size=20,
                max_overflow=10,
                pool_pre_ping=True,
                **kwargs,
            )
            msg = "OracleDB connected: {host}:{port}".format(**self._config)
            print(msg)
        except Exception as e:
            LOG.error(e)
            raise RuntimeError("Failed to connect to OracleDB") from e

    @property
    def connection(self):
        return self.engine

    def execute_query(self, sql, *args, **kwargs):
        """Execute a SQL statement and return the cursor result."""
        with self.engine.connect() as conn:
            cur = conn.execute(text(sql), *args, **kwargs)
            return cur

    @property
    def connection_str(self) -> str:
        """Return a human-readable connection string."""
        return "oracle://{user}@{host}:{port}".format(**self._config)

connection_str property

Return a human-readable connection string.

__init__(db_config, **kwargs)

Build an Oracle connection pool from the provided config.

Source code in aloha/db/oracle.py
def __init__(self, db_config, **kwargs):
    """Build an Oracle connection pool from the provided config."""
    """example of db_config:
    {
        "host": "192.168.1.100",
        "port": 1521,
        "user": "PT_INDEX",
        "password": "vault_key_or_plain",
        "service_name": "orcl",   # 推荐使用 service_name
        "sid": "orcl",            # 或使用 sid
        "vault_type": "...",
        "vault_config": {...},
        "lib_dir": "/opt/oracle/instantclient"  # optional, use THICK mode if defined.
    }
    """

    password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config"))
    self._config = {
        "host": db_config["host"],
        "port": db_config["port"],
        "user": db_config["user"],
        "password": password_vault.get_password(db_config.get("password")),
    }

    if "lib_dir" in db_config:  # use Thick mode
        try:
            oracledb.init_oracle_client(lib_dir=db_config["lib_dir"])
            LOG.info("Oracle client initialized in THICK mode from: {}".format(db_config["lib_dir"]))
        except Exception as e:
            LOG.warning(f"Warning: {e}")
            raise RuntimeError(f"Failed to initialize Oracle client: {e}") from e

    service_name = db_config.get("service_name")
    sid = db_config.get("sid")

    if service_name:  # using service_name (recommended)
        dsn = oracledb.makedsn(db_config["host"], db_config["port"], service_name=service_name)
    elif sid:  # using SID
        dsn = oracledb.makedsn(db_config["host"], db_config["port"], sid=sid)
    else:
        raise ValueError("Oracle config must specify service_name or sid")

    self._config["dsn"] = dsn
    try:
        self.engine = create_engine(
            "oracle+oracledb://{user}:{password}@".format(**self._config),
            connect_args={"dsn": dsn},
            pool_size=20,
            max_overflow=10,
            pool_pre_ping=True,
            **kwargs,
        )
        msg = "OracleDB connected: {host}:{port}".format(**self._config)
        print(msg)
    except Exception as e:
        LOG.error(e)
        raise RuntimeError("Failed to connect to OracleDB") from e

execute_query(sql, *args, **kwargs)

Execute a SQL statement and return the cursor result.

Source code in aloha/db/oracle.py
def execute_query(self, sql, *args, **kwargs):
    """Execute a SQL statement and return the cursor result."""
    with self.engine.connect() as conn:
        cur = conn.execute(text(sql), *args, **kwargs)
        return cur

PostgreSQL connection helpers.

PostgresOperator

Create and use a SQLAlchemy-backed PostgreSQL connection.

Source code in aloha/db/postgres.py
class PostgresOperator:
    """Create and use a SQLAlchemy-backed PostgreSQL connection."""

    def __init__(self, db_config, **kwargs):
        """Build a PostgreSQL connection pool from the database config."""
        password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config"))
        self._config = {
            "host": db_config["host"],
            "port": db_config["port"],
            "user": db_config["user"],
            "password": password_vault.get_password(db_config.get("password")),
            "dbname": db_config["dbname"],
        }
        connect_args = {}
        if "schema" in db_config:
            connect_args["options"] = "-csearch_path={}".format(db_config["schema"])

        try:
            self.engine = create_engine(
                "postgresql+psycopg://{user}:{password}@{host}:{port}/{dbname}".format(**self._config),
                connect_args=connect_args,
                client_encoding="utf8",
                pool_size=20,
                max_overflow=10,
                pool_pre_ping=True,
                **kwargs,
            )
            LOG.debug("PostgresSQL connected: {host}:{port}/{dbname}".format(**self._config))
        except Exception as e:
            LOG.error(e)
            raise RuntimeError("Failed to connect to PostgresSQL") from e

    @property
    def connection(self):
        return self.engine

    def execute_query(self, sql, *args, **kwargs):
        """Execute a SQL statement and return the cursor result."""
        with self.engine.connect() as conn:
            cur = conn.execute(text(sql), *args, **kwargs)
            return cur

    @property
    def connection_str(self) -> str:
        """Return a human-readable connection string."""
        return "postgresql://{user}:{password}@{host}:{port}/{dbname}".format(**self._config)

connection_str property

Return a human-readable connection string.

__init__(db_config, **kwargs)

Build a PostgreSQL connection pool from the database config.

Source code in aloha/db/postgres.py
def __init__(self, db_config, **kwargs):
    """Build a PostgreSQL connection pool from the database config."""
    password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config"))
    self._config = {
        "host": db_config["host"],
        "port": db_config["port"],
        "user": db_config["user"],
        "password": password_vault.get_password(db_config.get("password")),
        "dbname": db_config["dbname"],
    }
    connect_args = {}
    if "schema" in db_config:
        connect_args["options"] = "-csearch_path={}".format(db_config["schema"])

    try:
        self.engine = create_engine(
            "postgresql+psycopg://{user}:{password}@{host}:{port}/{dbname}".format(**self._config),
            connect_args=connect_args,
            client_encoding="utf8",
            pool_size=20,
            max_overflow=10,
            pool_pre_ping=True,
            **kwargs,
        )
        LOG.debug("PostgresSQL connected: {host}:{port}/{dbname}".format(**self._config))
    except Exception as e:
        LOG.error(e)
        raise RuntimeError("Failed to connect to PostgresSQL") from e

execute_query(sql, *args, **kwargs)

Execute a SQL statement and return the cursor result.

Source code in aloha/db/postgres.py
def execute_query(self, sql, *args, **kwargs):
    """Execute a SQL statement and return the cursor result."""
    with self.engine.connect() as conn:
        cur = conn.execute(text(sql), *args, **kwargs)
        return cur

Redis connection helpers.

RedisOperator

Create Redis connections with version-checked redis-py.

Source code in aloha/db/redis.py
class RedisOperator:
    """Create Redis connections with version-checked redis-py."""

    def __init__(self, config):
        """Normalize Redis connection settings and build connection metadata."""
        self._check_redis_version()

        password_vault = PasswordVault.get_vault(config.get("vault_type"), config.get("vault_config"))
        _config = {
            "host": config["host"],
            "port": config.get("port", "6379"),
            "password": password_vault.get_password(config.get("password", None)),
            "decode_responses": config.get("decode_responses", True),
            "retry_on_timeout": True,
            "max_connections": config.get("max_connections", 1000),
            "socket_timeout": 3,
            "socket_connect_timeout": 1,
        }
        if "db" in config:
            _config["db"] = config["db"]
        self._config = _config

        self._pool = None

    @staticmethod
    def _check_redis_version() -> bool:
        """Ensure a redis-py version new enough for the helpers is installed."""
        ver_min = version.parse("4.1.0")
        valid = False
        try:
            ver_cur = version.parse(redis.__version__)
            if ver_cur >= ver_min:
                valid = True
                LOG.debug(f"Using redis version = {redis.__version__}")
        except Exception as e:  # noqa: BLE001
            LOG.error("Failed to obtain redis version!")
            LOG.error(str(e))

        if not valid:
            msg = "Invalid version of `redis-py`, version >4.1.0 required!"
            LOG.fatal(msg)
            raise ImportError(msg)

        return valid

    @property
    def connection_generic(self):
        """Return a standard Redis client."""
        LOG.debug("StrictRedis connection info: {host}:{port}".format(**self._config))

        if self._pool is None:
            self._pool = redis.ConnectionPool()
        return redis.Redis(connection_pool=self._pool, **self._config)

    @property
    def connection_cluster(self):
        """Return a Redis Cluster client."""
        LOG.debug("RedisCluster connection info: {host}:{port}".format(**self._config))
        return redis.RedisCluster(**self._config)

connection_cluster property

Return a Redis Cluster client.

connection_generic property

Return a standard Redis client.

__init__(config)

Normalize Redis connection settings and build connection metadata.

Source code in aloha/db/redis.py
def __init__(self, config):
    """Normalize Redis connection settings and build connection metadata."""
    self._check_redis_version()

    password_vault = PasswordVault.get_vault(config.get("vault_type"), config.get("vault_config"))
    _config = {
        "host": config["host"],
        "port": config.get("port", "6379"),
        "password": password_vault.get_password(config.get("password", None)),
        "decode_responses": config.get("decode_responses", True),
        "retry_on_timeout": True,
        "max_connections": config.get("max_connections", 1000),
        "socket_timeout": 3,
        "socket_connect_timeout": 1,
    }
    if "db" in config:
        _config["db"] = config["db"]
    self._config = _config

    self._pool = None

SQLite connection helpers.

SqliteOperator

Create and use a SQLAlchemy-backed SQLite connection.

Source code in aloha/db/sqlite.py
class SqliteOperator:
    """Create and use a SQLAlchemy-backed SQLite connection."""

    def __init__(self, db_config, **kwargs):
        """Build a SQLite or SQLCipher engine from the provided config."""
        self._connection_pattern = "sqlite://{dbname}"
        dbname = db_config.get("dbname", "")
        if len(dbname) > 0:
            dbname = f"/{dbname}"
        self._config = {"dbname": dbname}

        if "password" in db_config:
            try:
                import sqlcipher3
            except ImportError:
                raise RuntimeError("Python package required for encrypted sqlite3: sqlcipher3-binary")
            LOG.debug(f"Version of sqlcipher3 = {sqlcipher3.sqlite_version}")
            password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config"))
            password = password_vault.get_password(db_config.get("password", None))
            self._config["password"] = password
            self._connection_pattern = "sqlite+pysqlcipher://:{password}@/{dbname}"
        else:
            LOG.debug(f"Version of sqlite = {sqlite3.sqlite_version}")

        try:
            self.db = create_engine(self._connection_pattern.format(**self._config), **kwargs)
            LOG.debug(f"Sqlite connected: {self.connection_str}")
        except Exception as e:
            LOG.exception(e)
            raise RuntimeError("Failed to connect to sqlite") from e

    @property
    def connection(self):
        return self.db

    def execute_query(self, sql, *args, **kwargs):
        """Execute a SQL statement and return the cursor result."""
        with self.db.connect() as conn:
            cur = conn.execute(text(sql), *args, **kwargs)
            return cur

    @property
    def connection_str(self) -> str:
        """Return the SQLAlchemy connection URL used by the engine."""
        return self._connection_pattern.format(**self._config)

connection_str property

Return the SQLAlchemy connection URL used by the engine.

__init__(db_config, **kwargs)

Build a SQLite or SQLCipher engine from the provided config.

Source code in aloha/db/sqlite.py
def __init__(self, db_config, **kwargs):
    """Build a SQLite or SQLCipher engine from the provided config."""
    self._connection_pattern = "sqlite://{dbname}"
    dbname = db_config.get("dbname", "")
    if len(dbname) > 0:
        dbname = f"/{dbname}"
    self._config = {"dbname": dbname}

    if "password" in db_config:
        try:
            import sqlcipher3
        except ImportError:
            raise RuntimeError("Python package required for encrypted sqlite3: sqlcipher3-binary")
        LOG.debug(f"Version of sqlcipher3 = {sqlcipher3.sqlite_version}")
        password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config"))
        password = password_vault.get_password(db_config.get("password", None))
        self._config["password"] = password
        self._connection_pattern = "sqlite+pysqlcipher://:{password}@/{dbname}"
    else:
        LOG.debug(f"Version of sqlite = {sqlite3.sqlite_version}")

    try:
        self.db = create_engine(self._connection_pattern.format(**self._config), **kwargs)
        LOG.debug(f"Sqlite connected: {self.connection_str}")
    except Exception as e:
        LOG.exception(e)
        raise RuntimeError("Failed to connect to sqlite") from e

execute_query(sql, *args, **kwargs)

Execute a SQL statement and return the cursor result.

Source code in aloha/db/sqlite.py
def execute_query(self, sql, *args, **kwargs):
    """Execute a SQL statement and return the cursor result."""
    with self.db.connect() as conn:
        cur = conn.execute(text(sql), *args, **kwargs)
        return cur