Intro to the core interfaces of mloda¶

# Load all available plugins into the python environment
from mloda_core.abstract_plugins.plugin_loader.plugin_loader import PluginLoader

plugin_loader = PluginLoader.all()


# Since there are potentially many plugins loaded, we'll focus on specific categories for clarity.
# Here, we demonstrate by listing the available 'read' and 'sql' plugins.
print(plugin_loader.list_loaded_modules("read"))
print(plugin_loader.list_loaded_modules("sql"))

# Step 3: Request Data Using the Defined Access Collection and Desired Features
from mloda_core.api.request import mlodaAPI
from mloda_plugins.compute_framework.base_implementations.pyarrow.table import PyarrowTable


all_features = order_features + payment_features + location_features + categorical_features

# Retrieve data based on the specified feature list and access collection
result = mlodaAPI.run_all(
    all_features, data_access_collection=data_access_collection, compute_frameworks={PyarrowTable}
)

# Display the first five entries of each result table and its type
for data in result:
    print(data[:2], type(data))

# The data is initially loaded as a Pyarrow table. However, we can easily load it also as a PandasDataframe.
from mloda_plugins.compute_framework.base_implementations.pandas.dataframe import PandasDataframe

# Request data using the Pandas compute framework
result = mlodaAPI.run_all(
    all_features, data_access_collection=data_access_collection, compute_frameworks={PandasDataframe}
)

# Display the first five entries of each result table and its type
for data in result:
    print(data[:2], type(data))

# Define features with specific compute frameworks
order_id = Feature(name="order_id", compute_framework="PandasDataframe")
product_id = Feature(name="product_id", compute_framework="PyarrowTable")
specific_framework_feature_list: List[Feature | str] = [order_id, product_id]

# Request data for the defined features
result = mlodaAPI.run_all(specific_framework_feature_list, data_access_collection=data_access_collection)

# Display the first few rows and data types of the results
for res in result:
    print("The resulting data structure differs based on the compute framework:")
    print("\n", res[:3], type(res))

# Demonstrating mloda's Flexibility with Different Data Technologies

# Import required modules
from typing import Any, List, Optional, Set

from mloda_core.abstract_plugins.abstract_feature_group import AbstractFeatureGroup
from mloda_core.abstract_plugins.components.feature_name import FeatureName
from mloda_core.abstract_plugins.components.feature_set import FeatureSet
from mloda_core.abstract_plugins.components.index.index import Index
from mloda_core.abstract_plugins.components.link import JoinType, Link
from mloda_core.abstract_plugins.components.options import Options
from mloda_plugins.feature_group.input_data.read_file_feature import ReadFileFeature


# Define the index for the join
index = Index(("order_id",))


# Extend ReadFileFeature to provide index columns
class ReadFileFeatureJoin(ReadFileFeature):
    @classmethod
    def index_columns(cls) -> Optional[List[Index]]:
        return [index]


# Define the link between the features
link = Link(jointype=JoinType.INNER, left=(ReadFileFeatureJoin, index), right=(ReadFileFeatureJoin, index))


# Create an example feature group to demonstrate joining
class ExampleMlLifeCycleJoin(AbstractFeatureGroup):
    # Define input features with different compute frameworks
    def input_features(self, options: Options, feature_name: FeatureName) -> Optional[Set[Feature]]:
        quantity = Feature(name="quantity", compute_framework="PandasDataframe")
        product_id = Feature(name="product_id", compute_framework="PyarrowTable")
        return {product_id, quantity}

    # Perform calculations on the joined data
    @classmethod
    def calculate_feature(cls, data: Any, features: FeatureSet) -> Any:
        print(
            "Data from two different sources is now combined into one feature within one data technology: \n",
            data,
            type(data),
            "\n",
        )
        return {"ExampleMlLifeCycleJoin": [1, 2, 3]}


# Run the pipeline
result = mlodaAPI.run_all(["ExampleMlLifeCycleJoin"], data_access_collection=data_access_collection, links={link})


# Display the final result
print(
    "Final result: ",
    result[0],
    "\nNote: As no specific compute framework was defined for the result, the output could be in either format.",
)

# Summary: mloda's abstraction layer enables complex process pipelines that handle different data technologies.
# This decouples processes from the underlying data structure, ensuring flexibility and scalability.