support API updates & docs refactor

.github/workflows/ci.yaml

@@ -31,8 +31,8 @@ jobs:
 
       - name: Run linting
         run: |
-          uvx ruff format --check
-          uvx ruff check
+          uvx ruff format --check --exclude "docs/**"
+          uvx ruff check --exclude "docs/**"
 
       - name: Run tests
         run: |
@@ -45,7 +45,7 @@ jobs:
 
       - name: Run docs generation
         run: |
-          uv run python docs-render.py
+          uv run python docs/render.py
           if [ -n "$(git status --porcelain docs/index.md)" ]; then
             echo "Error: docs/index.md has uncommitted changes after running render.py"
             echo "Please run 'python render.py' locally and commit the changes"

README.md

@@ -20,7 +20,7 @@ The `nortech-python` library is the official Python client for interacting with
 
 The `Nortech` class serves as the primary entry point for the library. It encapsulates the core functionalities and provides a unified interface to interact with the Nortech API. It has 3 main components:
 
-- **Metadata**: Access and manage metadata such as workspaces, assets, divisions, units, devices, and signals.
+- **Metadata**: Access and manage metadata such as workspaces, assets, divisions, units and signals.
 - **Datatools**: Fetch and manipulate signal data, supporting both Pandas and Polars DataFrames, time window queries, and signal filtering.
 - **Derivers**: Create and manage derivers, which allow computation of new signals based on existing ones. This includes creating deriver schemas, deploying derivers, managing configurations, and testing locally.
 
@@ -89,145 +89,4 @@ These functions return a `PaginatedResponse` object containing 3 functions:
 
 `PaginatedResponse` also has a `next_pagination_options` method that returns a `PaginationOptions`, which can also be used to fetch the next page.
 
-## Examples
-
-### nortech.datatools
-
-To get a DataFrame with the requested signals:
-
-1. Go to your `Signal Search` interface.
-2. Select the desired signals.
-3. Select the `DataTools` exported columns and copy the resulting `search_json`.
-4. Use the `signals` field and speficy a `TimeWindow` as in the examples bellow.
-
-##### Pandas DataFrame
-
-In order to get a [pandas](https://pandas.pydata.org/docs/) DataFrame use the `get_df` handler:
-
-```python
-from datetime import datetime
-
-from nortech import Nortech
-from nortech.core.values.signal import SignalInput, SignalInputDict
-from nortech.datatools.values.windowing import TimeWindow
-
-# Initialize the Nortech client
-nortech = Nortech()
-
-# Define signals to download
-signal1: SignalInputDict = {
-    "workspace": "workspace1",
-    "asset": "asset1",
-    "division": "division1",
-    "unit": "unit1",
-    "signal": "signal1",
-}
-signal2 = 789  # Signal ID
-signal3 = SignalInput(workspace="workspace2", asset="asset2", division="division2", unit="unit2", signal="signal2")
-
-# Define the time window for data download
-my_time_window = TimeWindow(start=datetime(2023, 1, 1), end=datetime(2023, 1, 31))
-
-# Call the get_df function
-df = nortech.datatools.pandas.get_df(
-    signals=[signal1, signal2, signal3],
-    time_window=my_time_window,
-)
-
-print(df.columns)
-# Output
-# [
-#     'timestamp',
-#     'workspace_1/asset_1/division_1/unit_1/signal_1',
-#     'workspace_1/asset_1/division_1/unit_1/signal_2',
-#     'workspace_2/asset_2/division_2/unit_2/signal_3'
-# ]
-```
-
-##### Polars DataFrame
-
-In order to get a [polars](https://pola-rs.github.io/polars/py-polars/html/reference/) DataFrame use the `get_df`:
-
-```python
-from datetime import datetime
-
-from nortech import Nortech
-from nortech.core.values.signal import SignalInput, SignalInputDict
-from nortech.datatools.values.windowing import TimeWindow
-
-# Initialize the Nortech client
-nortech = Nortech()
-
-# Define signals to download
-signal1: SignalInputDict = {
-    "workspace": "workspace1",
-    "asset": "asset1",
-    "division": "division1",
-    "unit": "unit1",
-    "signal": "signal1",
-}
-signal2 = 789  # Signal ID
-signal3 = SignalInput(workspace="workspace2", asset="asset2", division="division2", unit="unit2", signal="signal2")
-
-# Define the time window for data download
-my_time_window = TimeWindow(start=datetime(2023, 1, 1), end=datetime(2023, 1, 31))
-
-# Call the get_df function
-polars_df = nortech.datatools.polars.get_df(
-    signals=[signal1, signal2, signal3],
-    time_window=my_time_window,
-)
-
-print(polars_df.columns)
-# Output:
-# [
-#     'timestamp',
-#     'workspace_1/asset_1/division_1/unit_1/signal_1',
-#     'workspace_1/asset_1/division_1/unit_1/signal_2',
-#     'workspace_2/asset_2/division_2/unit_2/signal_3'
-# ]
-```
-
-##### Polars LazyFrame
-
-In order to get a [polars](https://pola-rs.github.io/polars/py-polars/html/reference/) LazyFrame use the `get_lazy_df`:
-
-```python
-from datetime import datetime
-
-from nortech import Nortech
-from nortech.core.values.signal import SignalInput, SignalInputDict
-from nortech.datatools.values.windowing import TimeWindow
-
-# Initialize the Nortech client
-nortech = Nortech()
-
-# Define signals to download
-signal1: SignalInputDict = {
-    "workspace": "workspace1",
-    "asset": "asset1",
-    "division": "division1",
-    "unit": "unit1",
-    "signal": "signal1",
-}
-signal2 = 789  # Signal ID
-signal3 = SignalInput(workspace="workspace2", asset="asset2", division="division2", unit="unit2", signal="signal2")
-
-# Define the time window for data download
-my_time_window = TimeWindow(start=datetime(2023, 1, 1), end=datetime(2023, 1, 31))
-
-# Call the get_lazy_df function
-lazy_polars_df = nortech.datatools.polars.get_lazy_df(
-    signals=[signal1, signal2, signal3],
-    time_window=my_time_window,
-)
-
-print(lazy_polars_df.columns)
-# Output:
-# [
-#     'timestamp',
-#     'workspace_1/asset_1/division_1/unit_1/signal_1',
-#     'workspace_1/asset_1/division_1/unit_1/signal_2',
-#     'workspace_2/asset_2/division_2/unit_2/signal_3'
-# ]
-```
+For comprehensive documentation including all available methods, parameters, and detailed examples, see the [Documentation](docs/index.md).

docs/examples/datatools/Download/download_data.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+
+from nortech import Nortech
+from nortech.datatools.values.windowing import TimeWindow
+from nortech.metadata.values.signal import SignalInput, SignalInputDict
+
+# Initialize the Nortech client
+nortech = Nortech()
+
+# Define signals to download
+signal1: SignalInputDict = {
+    "workspace": "workspace1",
+    "asset": "asset1",
+    "division": "division1",
+    "unit": "unit1",
+    "signal": "signal1",
+}
+signal2 = 789  # Signal ID
+signal3 = SignalInput(workspace="workspace2", asset="asset2", division="division2", unit="unit2", signal="signal2")
+
+fetched_signals = nortech.metadata.signal.list(  # Fetched signals
+    {"workspace": "workspace3", "asset": "asset3", "division": "division3", "unit": "unit3"}
+).data
+
+# Define the time window for data download
+my_time_window = TimeWindow(start=datetime(2023, 1, 1), end=datetime(2023, 1, 31))
+
+# Specify the output path and file format
+output_path = "path/to/output"
+file_format = "parquet"
+
+# Call the download_data function with manually defined signals or fetched signals
+nortech.datatools.download.download_data(
+    signals=[signal1, signal2, signal3] + fetched_signals,
+    time_window=my_time_window,
+    output_path=output_path,
+    file_format=file_format,
+)

docs/examples/datatools/Pandas/get_df.py

@@ -0,0 +1,42 @@
+from datetime import datetime
+
+from nortech import Nortech
+from nortech.datatools.values.windowing import TimeWindow
+from nortech.metadata.values.signal import SignalInput, SignalInputDict
+
+# Initialize the Nortech client
+nortech = Nortech()
+
+# Define signals to download
+signal1: SignalInputDict = {
+    "workspace": "workspace1",
+    "asset": "asset1",
+    "division": "division1",
+    "unit": "unit1",
+    "signal": "signal1",
+}
+signal2 = 789  # Signal ID
+signal3 = SignalInput(workspace="workspace2", asset="asset2", division="division2", unit="unit2", signal="signal2")
+
+fetched_signals = nortech.metadata.signal.list(  # Fetched signals
+    {"workspace": "workspace3", "asset": "asset3", "division": "division3", "unit": "unit3"}
+).data
+
+# Define the time window for data download
+my_time_window = TimeWindow(start=datetime(2023, 1, 1), end=datetime(2023, 1, 31))
+
+# Call the get_df function with manually defined signals or fetched signals
+df = nortech.datatools.pandas.get_df(
+    signals=[signal1, signal2, signal3] + fetched_signals,
+    time_window=my_time_window,
+)
+
+print(df.columns)
+# [
+#     "timestamp",
+#     "workspace_1/asset_1/division_1/unit_1/signal_1",
+#     "workspace_1/asset_1/division_1/unit_1/signal_2",
+#     "workspace_2/asset_2/division_2/unit_2/signal_3",
+#     "workspace_3/asset_3/division_3/unit_3/signal_4",
+#     "workspace_3/asset_3/division_3/unit_3/signal_5",
+# ]

docs/examples/datatools/Polars/get_df.py

@@ -0,0 +1,42 @@
+from datetime import datetime
+
+from nortech import Nortech
+from nortech.datatools.values.windowing import TimeWindow
+from nortech.metadata.values.signal import SignalInput, SignalInputDict
+
+# Initialize the Nortech client
+nortech = Nortech()
+
+# Define signals to download
+signal1: SignalInputDict = {
+    "workspace": "workspace1",
+    "asset": "asset1",
+    "division": "division1",
+    "unit": "unit1",
+    "signal": "signal1",
+}
+signal2 = 789  # Signal ID
+signal3 = SignalInput(workspace="workspace2", asset="asset2", division="division2", unit="unit2", signal="signal2")
+
+fetched_signals = nortech.metadata.signal.list(  # Fetched signals
+    {"workspace": "workspace3", "asset": "asset3", "division": "division3", "unit": "unit3"}
+).data
+
+# Define the time window for data download
+my_time_window = TimeWindow(start=datetime(2023, 1, 1), end=datetime(2023, 1, 31))
+
+# Call the get_df function with manually defined signals or fetched signals
+df = nortech.datatools.polars.get_df(
+    signals=[signal1, signal2, signal3] + fetched_signals,
+    time_window=my_time_window,
+)
+
+print(df.columns)
+# [
+#     "timestamp",
+#     "workspace_1/asset_1/division_1/unit_1/signal_1",
+#     "workspace_1/asset_1/division_1/unit_1/signal_2",
+#     "workspace_2/asset_2/division_2/unit_2/signal_3",
+#     "workspace_3/asset_3/division_3/unit_3/signal_4",
+#     "workspace_3/asset_3/division_3/unit_3/signal_5",
+# ]

docs/examples/datatools/Polars/get_lazy_df.py

@@ -0,0 +1,42 @@
+from datetime import datetime
+
+from nortech import Nortech
+from nortech.datatools.values.windowing import TimeWindow
+from nortech.metadata.values.signal import SignalInput, SignalInputDict
+
+# Initialize the Nortech client
+nortech = Nortech()
+
+# Define signals to download
+signal1: SignalInputDict = {
+    "workspace": "workspace1",
+    "asset": "asset1",
+    "division": "division1",
+    "unit": "unit1",
+    "signal": "signal1",
+}
+signal2 = 789  # Signal ID
+signal3 = SignalInput(workspace="workspace2", asset="asset2", division="division2", unit="unit2", signal="signal2")
+
+fetched_signals = nortech.metadata.signal.list(  # Fetched signals
+    {"workspace": "workspace3", "asset": "asset3", "division": "division3", "unit": "unit3"}
+).data
+
+# Define the time window for data download
+my_time_window = TimeWindow(start=datetime(2023, 1, 1), end=datetime(2023, 1, 31))
+
+# Call the get_df function with manually defined signals or fetched signals
+df = nortech.datatools.polars.get_lazy_df(
+    signals=[signal1, signal2, signal3] + fetched_signals,
+    time_window=my_time_window,
+)
+
+print(df.columns)
+# [
+#     "timestamp",
+#     "workspace_1/asset_1/division_1/unit_1/signal_1",
+#     "workspace_1/asset_1/division_1/unit_1/signal_2",
+#     "workspace_2/asset_2/division_2/unit_2/signal_3",
+#     "workspace_3/asset_3/division_3/unit_3/signal_4",
+#     "workspace_3/asset_3/division_3/unit_3/signal_5",
+# ]

docs/examples/derivers/Derivers/__init__.py

@@ -0,0 +1,55 @@
+# To define a deriver, you need to create a class that inherits from the Deriver class.
+# The class must have two inner classes: Inputs and Outputs.
+# The Inputs class must inherit from DeriverInputs and the Outputs class must inherit from DeriverOutputs.
+# The Inputs class must define the inputs of the deriver.
+# The Outputs class must define the outputs of the deriver.
+# The run method must be defined and return a bytewax stream.
+
+from __future__ import annotations
+
+import bytewax.operators as op
+
+from nortech.derivers import Deriver, DeriverInput, DeriverInputs, DeriverOutput, DeriverOutputs
+
+
+class MyDeriver(Deriver):
+    class Inputs(DeriverInputs):
+        input_1: float | None = DeriverInput(
+            workspace="workspace1", asset="asset1", division="division1", unit="unit1", signal="signal1"
+        )
+        input_2: float | None = DeriverInput(
+            workspace="workspace2", asset="asset2", division="division2", unit="unit2", signal="signal2"
+        )
+
+    class Outputs(DeriverOutputs):
+        output_1: float = DeriverOutput(
+            workspace="workspace1",
+            asset="asset1",
+            division="division1",
+            unit="unit1",
+            signal="new_signal1",
+            description="output_1",
+            long_description="output_1_long_description",
+            physical_unit="m/s",
+        )
+        output_2: str = DeriverOutput(
+            workspace="workspace2",
+            asset="asset2",
+            division="division2",
+            unit="unit2",
+            signal="new_signal2",
+            description="output_2",
+            long_description="output_2_long_description",
+            physical_unit="m/s",
+        )
+
+    def run(self, inputs: op.Stream[Inputs]) -> op.Stream[Outputs]:
+        return op.map(
+            "",
+            inputs,
+            lambda _input: self.Outputs(
+                timestamp=_input.timestamp,
+                output_1=_input.input_1 or 0,
+                output_2=str(_input.input_2),
+            ),
+        )