[HWORKS-2317] Add dataset api list function (#643)

Author: robzor92

java/beam/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>4.5.0-RC5</version>
+    <version>4.5.0-RC6</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/flink/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>4.5.0-RC5</version>
+    <version>4.5.0-RC6</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/hsfs/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>4.5.0-RC5</version>
+    <version>4.5.0-RC6</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/pom.xml

@@ -7,7 +7,7 @@
     <groupId>com.logicalclocks</groupId>
     <artifactId>hsfs-parent</artifactId>
     <packaging>pom</packaging>
-    <version>4.5.0-RC5</version>
+    <version>4.5.0-RC6</version>
     <modules>
         <module>hsfs</module>
         <module>spark</module>

java/spark/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>hsfs-parent</artifactId>
         <groupId>com.logicalclocks</groupId>
-        <version>4.5.0-RC5</version>
+        <version>4.5.0-RC6</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

python/auto_doc.py

@@ -152,8 +152,6 @@
                 "delete",
                 "path_exists",
                 "rm",
-                "list",
-                "list_files",
             ],
         ),
     },

python/hopsworks_common/core/dataset.py

@@ -0,0 +1,63 @@
+#
+#   Copyright 2025 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+import humps
+
+
+class Dataset:
+    def __init__(
+        self,
+        id,
+        name,
+        dataset_type,
+        attributes: Dict[str, Any],
+        **kwargs,
+    ) -> None:
+        self._id = id
+        self._name = name
+        self._description = kwargs.get("description", None)
+        self._dataset_type = dataset_type
+        self._path = attributes["path"]
+
+    @classmethod
+    def from_response_json(cls, json_dict: Dict[str, Any]) -> List[Dataset]:
+        json_decamelized = humps.decamelize(json_dict)["items"]
+        for dataset in json_decamelized:
+            _ = dataset.pop("type", None)
+        return [cls(**dataset) for dataset in json_decamelized]
+    @property
+    def id(self) -> str:
+        return self._id
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def description(self) -> str:
+        return self._description
+
+    @property
+    def dataset_type(self) -> str:
+        return self._dataset_type
+
+    @property
+    def path(self) -> str:
+        return self._path

python/hopsworks_common/core/dataset_api.py

@@ -24,11 +24,11 @@
 import shutil
 import time
 from concurrent.futures import ThreadPoolExecutor, wait
-from typing import Literal, Optional, Union
+from typing import List, Literal, Optional, Type, Union
 
 from hopsworks_common import client, tag, usage, util
 from hopsworks_common.client.exceptions import DatasetException, RestAPIError
-from hopsworks_common.core import inode
+from hopsworks_common.core import dataset, inode
 from tqdm.auto import tqdm
 
 
@@ -638,68 +638,80 @@ def upload_feature_group(self, feature_group, path, dataframe):
             chunk_number += 1
 
     @usage.method_logger
-    def list_files(self, path: str, offset: int, limit: int):
-        """**Deprecated**
+    def list(self, path: str, offset: int = 0, limit: int = 1000) -> List[str]:
+        """List the files and directories from a path in the Hopsworks Filesystem.
+
+        ```python
+
+        import hopsworks
+
+        project = hopsworks.login()
+
+        dataset_api = project.get_dataset_api()
+
+        # list all files in the Resources dataset
+        files = dataset_api.list("/Resources")
+
+        # list all datasets in the project
+        files = dataset_api.list("/")
+
+        ```
+        # Arguments
+            path: path in Hopsworks filesystem to the directory
+            offset: the number of entities to skip
+            limit: max number of the returned entities
+        # Returns
+            `list[str]`: List of path to files and directories in the provided path
+        # Raises
+            `hopsworks.client.exceptions.RestAPIError`: If the backend encounters an error when handling the request
+        """
+        _client = client.get_instance()
+        # Normalize path so we can check if the path refers to the root or not
+        # That is needed as different backend entities are returned depending on if it is a top level dataset or a subdirectory
+        normalized_path = os.path.normpath(path)
+        if normalized_path == "/":
+            normalized_path = ""
+            cls = dataset.Dataset
+        else:
+            cls = inode.Inode
+
+        count, items = self._list_dataset_path(normalized_path, cls, offset=offset, limit=limit)
 
+        files = []
+        for item in items:
+            files.append(util.convert_to_project_rel_path(item.path, _client._project_name))
+        return files
+
+    @usage.method_logger
+    def _list_dataset_path(self, path: str, cls: Union[Type[dataset.Dataset], Type[inode.Inode]], offset: int = 0, limit: int = 1000, sort_by: str = "ID:asc"):
+        """
         List contents of a directory in the Hopsworks Filesystem.
 
         # Arguments
             path: path to the directory to list the contents of.
             offset: the number of Inodes to skip.
             limit: max number of the returned Inodes.
         # Returns
-            `tuple[int, list[hopsworks.core.inode.Inode]]`: count of Inodes in the directory and the list of them.
+            `tuple[int, tuple[int, list[inode.Inode]] | tuple[int, list[Dataset]]]`: count of Dataset or Inodes and objects
         """
         _client = client.get_instance()
         path_params = [
             "project",
             _client._project_id,
             "dataset",
-            path[(path.index("/", 10) + 1) :],
+            path,
         ]
         query_params = {
             "action": "listing",
             "offset": offset,
             "limit": limit,
-            "sort_by": "ID:asc",
+            "sort_by": sort_by,
+            "expand": "inodes",
         }
 
-        inode_lst = _client._send_request("GET", path_params, query_params)
-
-        return inode_lst["count"], inode.Inode.from_response_json(inode_lst)
-
-    @usage.method_logger
-    def list(
-        self,
-        remote_path: str,
-        sort_by: str | None = None,
-        offset: int = 0,
-        limit: int = 1000,
-    ):
-        """**Deprecated**
-
-        List contents of a directory in the Hopsworks Filesystem.
+        items = _client._send_request("GET", path_params, query_params)
 
-        # Arguments
-            remote_path: path to the directory to list the contents of.
-            sort_by: sort string, for example `"ID:asc"`.
-            offset: the number of entities to skip.
-            limit: max number of the returned entities.
-        """
-        # this method is probably to be merged with list_files
-        # they seem to handle paths differently and return different results, which prevents the merge at the moment (2024-09-03), due to the requirement of backwards-compatibility
-        _client = client.get_instance()
-        path_params = ["project", _client._project_id, "dataset", remote_path]
-        query_params = {
-            "action": "listing",
-            "sort_by": sort_by,
-            "limit": limit,
-            "offset": offset,
-        }
-        headers = {"content-type": "application/json"}
-        return _client._send_request(
-            "GET", path_params, headers=headers, query_params=query_params
-        )
+        return items["count"], cls.from_response_json(items)
 
     @usage.method_logger
     def read_content(self, path: str, dataset_type: str = "DATASET"):

python/hopsworks_common/core/inode.py

@@ -29,7 +29,13 @@ def __init__(
         tags: Optional[Dict[str, Any]] = None,
         **kwargs,
     ) -> None:
-        self._path = attributes["path"]
+        self._name = attributes.get("name", None)
+        self._dir = attributes.get("dir", False)
+        self._owner = attributes.get("owner", None)
+        self._path = attributes.get("path", None)
+        self._permission = attributes.get("permission", None)
+        self._modification_time = attributes.get("modification_time", None)
+        self._under_construction = attributes.get("under_construction", None)
 
     @classmethod
     def from_response_json(cls, json_dict: Dict[str, Any]) -> List[Inode]:
@@ -38,6 +44,30 @@ def from_response_json(cls, json_dict: Dict[str, Any]) -> List[Inode]:
             _ = inode.pop("type", None)
         return [cls(**inode) for inode in json_decamelized]
 
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def dir(self) -> str:
+        return self._dir
+
+    @property
+    def owner(self) -> str:
+        return self._owner
+
     @property
     def path(self) -> str:
         return self._path
+
+    @property
+    def permission(self) -> str:
+        return self._permission
+
+    @property
+    def modification_time(self) -> str:
+        return self._modification_time
+
+    @property
+    def under_construction(self) -> str:
+        return self._under_construction

python/hopsworks_common/util.py

@@ -438,6 +438,14 @@ def convert_to_abs(path, current_proj_name):
         return path
 
 
+def convert_to_project_rel_path(path, current_proj_name):
+    abs_project_prefix = "/Projects/{}".format(current_proj_name)
+    if path.startswith(abs_project_prefix):
+        return path.replace(abs_project_prefix, "")
+    else:
+        return path
+
+
 def validate_job_conf(config, project_name):
     # User is required to set the appPath programmatically after getting the configuration
     if (