Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"python.defaultInterpreterPath": ".venv/bin/python",
"[python]": {
"editor.defaultFormatter": "ms-python.black-formatter",
"editor.formatOnSave": true
}
}
7 changes: 2 additions & 5 deletions docs/reference/task_plugin.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,17 @@ and is registered via [TaskManager.with_plugin][fluid.scheduler.TaskManager.with

```python
from fluid.scheduler import TaskScheduler, task_manager_fastapi
from fluid.scheduler.db import TaskDbPlugin, with_task_history_router
from fluid.scheduler.db import TaskDbPlugin

task_manager = TaskScheduler(...)
task_manager.with_plugin(TaskDbPlugin(db))
app = task_manager_fastapi(task_manager)
with_task_history_router(app)
```

::: fluid.scheduler.TaskManagerPlugin

::: fluid.scheduler.db.TaskDbPlugin

::: fluid.scheduler.db.with_task_history_router

## Accessing the plugin from a task

[get_db_plugin][fluid.scheduler.db.get_db_plugin] retrieves the registered
Expand All @@ -48,7 +45,7 @@ async def report(context: TaskRun) -> None:

The following models are used when querying task run history via
[TaskDbPlugin.get_history][fluid.scheduler.db.TaskDbPlugin.get_history]
or the HTTP endpoints added by [with_task_history_router][fluid.scheduler.db.with_task_history_router].
or the HTTP endpoints.

They can be imported from `fluid.scheduler.db`:

Expand Down
10 changes: 4 additions & 6 deletions docs/tutorials/task_app.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,26 +120,24 @@ Register the plugin when building your task manager:

```python
from fluid.scheduler import TaskScheduler, task_manager_fastapi
from fluid.scheduler.db import TaskDbPlugin, with_task_history_router
from fluid.scheduler.db import TaskDbPlugin
from fluid.db import CrudDB

db = CrudDB.from_env()
task_manager = TaskScheduler(...)
task_manager.with_plugin(TaskDbPlugin(db))
app = task_manager_fastapi(task_manager)
with_task_history_router(app)
```

The plugin creates a `fluid_tasks` table (configurable via `table_name`) and
persists a row for each task run as it moves through its lifecycle states.
Tasks tagged with `skip_db` are excluded from persistence.

`with_task_history_router` mounts a `/history` router on the app with two endpoints:
The plugin mounts a `/tasks-history` router on the app with two endpoints:

| Method | Path | Description |
|--------|------|-------------|
| `GET` | `/history` | List task run history with optional filters |
| `GET` | `/history/{run_id}` | Fetch a single task run by ID |
| `GET` | `/tasks-history` | List task run history with optional filters |
| `GET` | `/tasks-history/{run_id}` | Fetch a single task run by ID |

The list endpoint accepts the following query parameters:

Expand Down
4 changes: 0 additions & 4 deletions examples/__main__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
import dotenv

dotenv.load_dotenv()

from examples.cli import task_manager_cli # isort:skip # noqa: E402

task_manager_cli()
4 changes: 4 additions & 0 deletions examples/cli.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import dotenv

from fluid.scheduler.cli import TaskManagerCLI

dotenv.load_dotenv()

task_manager_cli = TaskManagerCLI(
"examples.tasks:task_app", lazy_subcommands={"db": "examples.db.cli:cli"}
)
32 changes: 32 additions & 0 deletions examples/full_cli.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
from pathlib import Path

import dotenv

from fluid.db import CrudDB
from fluid.db.cli import DbGroup
from fluid.scheduler.cli import DEFAULT_COMMANDS, TaskManagerCLI
from fluid.scheduler.db import TaskDbPlugin

dotenv.load_dotenv()

MIGRATIONS_PATH = Path(__file__).parent / "tasks" / "migrations"


def create_cli() -> TaskManagerCLI:
from examples.tasks import task_app

# create the database for the db plugin
db = CrudDB.from_env(migration_path=MIGRATIONS_PATH, db_name="fluid_full_cli")
# create the client
task_manager_cli = TaskManagerCLI(
task_app(plugins=[TaskDbPlugin(db)]),
commands=list(DEFAULT_COMMANDS)
+ [DbGroup(db, help="Task database plugin management")],
help="Task Manager CLI with db plugin",
)
return task_manager_cli


if __name__ == "__main__":
task_manager_cli = create_cli()
task_manager_cli()
7 changes: 6 additions & 1 deletion examples/simple.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
import asyncio

from examples.tasks import task_scheduler
import dotenv

from fluid.utils import log

dotenv.load_dotenv()

if __name__ == "__main__":
from examples.tasks import task_scheduler

log.config()
asyncio.run(task_scheduler().run())
7 changes: 5 additions & 2 deletions examples/simple_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@

dotenv.load_dotenv()

from fluid.scheduler.cli import TaskManagerCLI # isort:skip # noqa: E402

if __name__ == "__main__":
from examples.tasks import task_app
from fluid.scheduler.cli import TaskManagerCLI

task_manager_cli = TaskManagerCLI(task_app())
task_manager_cli = TaskManagerCLI(
task_app(),
help="Simple Task Manager CLI with default commands",
)
task_manager_cli()
6 changes: 5 additions & 1 deletion examples/simple_fastapi.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
import dotenv
import uvicorn

from examples.tasks import task_app
from fluid.utils import log

dotenv.load_dotenv()

if __name__ == "__main__":
from examples.tasks import task_app

log.config()
uvicorn.run(task_app())
7 changes: 5 additions & 2 deletions examples/tasks/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,11 @@ def task_scheduler(
return task_manager


def task_app() -> FastAPI:
return task_manager_fastapi(task_scheduler(), app=FastAPI(title="Task Manager API"))
def task_app(plugins: Sequence[TaskManagerPlugin] | None = None) -> FastAPI:
return task_manager_fastapi(
task_scheduler(plugins=plugins),
app=FastAPI(title="Task Manager API"),
)


class Sleep(BaseModel):
Expand Down
1 change: 1 addition & 0 deletions examples/tasks/migrations/README
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Generic single-database configuration.
149 changes: 149 additions & 0 deletions examples/tasks/migrations/alembic.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
# A generic, single database configuration.

[alembic]
# path to migration scripts.
# this is typically a path given in POSIX (e.g. forward slashes)
# format, relative to the token %(here)s which refers to the location of this
# ini file
script_location = /home/luca/metablock/aio-fluid/examples/tasks/migrations

# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
# Uncomment the line below if you want the files to be prepended with date and time
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
# for all available tokens
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
# Or organize into date-based subdirectories (requires recursive_version_locations = true)
# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s

# sys.path path, will be prepended to sys.path if present.
# defaults to the current working directory. for multiple paths, the path separator
# is defined by "path_separator" below.
prepend_sys_path = .


# timezone to use when rendering the date within the migration file
# as well as the filename.
# If specified, requires the tzdata library which can be installed by adding
# `alembic[tz]` to the pip requirements.
# string value is passed to ZoneInfo()
# leave blank for localtime
# timezone =

# max length of characters to apply to the "slug" field
# truncate_slug_length = 40

# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false

# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false

# version location specification; This defaults
# to <script_location>/versions. When using multiple version
# directories, initial revisions must be specified with --version-path.
# The path separator used here should be the separator specified by "path_separator"
# below.
# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions

# path_separator; This indicates what character is used to split lists of file
# paths, including version_locations and prepend_sys_path within configparser
# files such as alembic.ini.
# The default rendered in new alembic.ini files is "os", which uses os.pathsep
# to provide os-dependent path splitting.
#
# Note that in order to support legacy alembic.ini files, this default does NOT
# take place if path_separator is not present in alembic.ini. If this
# option is omitted entirely, fallback logic is as follows:
#
# 1. Parsing of the version_locations option falls back to using the legacy
# "version_path_separator" key, which if absent then falls back to the legacy
# behavior of splitting on spaces and/or commas.
# 2. Parsing of the prepend_sys_path option falls back to the legacy
# behavior of splitting on spaces, commas, or colons.
#
# Valid values for path_separator are:
#
# path_separator = :
# path_separator = ;
# path_separator = space
# path_separator = newline
#
# Use os.pathsep. Default configuration used for new projects.
path_separator = os

# set to 'true' to search source files recursively
# in each "version_locations" directory
# new in Alembic version 1.10
# recursive_version_locations = false

# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8

# database URL. This is consumed by the user-maintained env.py script only.
# other means of configuring database URLs may be customized within the env.py
# file.
sqlalchemy.url = driver://user:pass@localhost/dbname


[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples

# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks = black
# black.type = console_scripts
# black.entrypoint = black
# black.options = -l 79 REVISION_SCRIPT_FILENAME

# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
# hooks = ruff
# ruff.type = module
# ruff.module = ruff
# ruff.options = check --fix REVISION_SCRIPT_FILENAME

# Alternatively, use the exec runner to execute a binary found on your PATH
# hooks = ruff
# ruff.type = exec
# ruff.executable = ruff
# ruff.options = check --fix REVISION_SCRIPT_FILENAME

# Logging configuration. This is also consumed by the user-maintained
# env.py script only.
[loggers]
keys = root,sqlalchemy,alembic

[handlers]
keys = console

[formatters]
keys = generic

[logger_root]
level = WARNING
handlers = console
qualname =

[logger_sqlalchemy]
level = WARNING
handlers =
qualname = sqlalchemy.engine

[logger_alembic]
level = INFO
handlers =
qualname = alembic

[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic

[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
Loading
Loading