diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1cd760472..cd3dc8be4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ repos: # Black: format Python code # https://github.com/psf/black/blob/master/.pre-commit-hooks.yaml - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 24.3.0 hooks: - id: black exclude: (versioneer.py|dataprofiler/_version.py|_docs/) @@ -21,7 +21,7 @@ repos: # Flake8: complexity and style checking # https://flake8.pycqa.org/en/latest/user/using-hooks.html - repo: https://github.com/pycqa/flake8 - rev: 4.0.1 + rev: 7.3.0 hooks: - id: flake8 additional_dependencies: [flake8-docstrings] @@ -50,29 +50,31 @@ repos: # requirements.txt h5py>=2.10.0, wheel>=0.33.1, - numpy<2.0.0, + 'numpy>=1.22.0,<3.0.0', 'pandas>=1.1.2,<3.0.0', python-dateutil>=2.7.5, pytz>=2020.1, pyarrow>=1.0.1, 'chardet>=3.0.4,<7.0.0', - fastavro>=1.0.0.post1, + fastavro>=1.1.0, python-snappy>=0.7.1, charset-normalizer>=1.3.6, psutil>=4.0.0, - scipy>=1.4.1, - requests>=2.28.1, + scipy>=1.10.0, + requests>=2.32.4, networkx>=2.5.1, typing-extensions>=3.10.0.2, HLL>=2.0.3, datasketches>=4.1.0, - boto3>=1.28.61, + packaging>=23.0, + boto3>=1.37.15, + urllib3>=2.5.0, # requirements-dev.txt - check-manifest>=0.48, - black==22.3.0, + check-manifest>=0.50, + black>=24.3.0, isort==5.12.0, - pre-commit==2.19.0, + pre-commit==4.3.0, tox==3.25.1, types-setuptools==67.7.0.1, types-python-dateutil==2.8.19.12, @@ -82,11 +84,9 @@ repos: # requirements-ml.txt scikit-learn>=0.23.2, - 'keras>=2.4.3,<=3.4.0', + 'keras>3.4.0,<4.0.0', rapidfuzz>=2.6.1, - "tensorflow>=2.6.4,<2.15.0; sys.platform != 'darwin'", - "tensorflow>=2.6.4,<2.15.0; sys_platform == 'darwin' and platform_machine != 'arm64'", - "tensorflow-macos>=2.6.4,<2.15.0; sys_platform == 'darwin' and platform_machine == 'arm64'", + "tensorflow>=2.16.0", tqdm>=4.0.0, # requirements-reports.txt @@ -101,18 +101,20 @@ repos: pytest-xdist>=2.1.0, pytest-forked>=1.3.0, toolz>=0.10.0, - 'memray>=1.7.0,<1.12.0', + 'memray>=1.18.0', ] # Check-manifest: ensures required non-Python files are included in MANIFEST.in # https://github.com/mgedmin/check-manifest/blob/master/.pre-commit-hooks.yaml - repo: https://github.com/mgedmin/check-manifest - rev: "0.48" + rev: "0.50" hooks: - id: check-manifest - additional_dependencies: ['h5py', 'wheel', 'future', 'numpy<2.0.0', 'pandas', - 'python-dateutil', 'pytz', 'pyarrow', 'chardet', 'fastavro', - 'python-snappy', 'charset-normalizer', 'psutil', 'scipy', 'requests', - 'networkx','typing-extensions', 'HLL', 'datasketches', 'boto3'] + additional_dependencies: ['h5py', 'wheel', 'future', 'numpy>=1.22.0,<3.0.0', + 'pandas', 'python-dateutil', 'pytz', 'pyarrow', 'chardet', + 'fastavro>=1.1.0', 'python-snappy', 'charset-normalizer', 'psutil', + 'scipy>=1.10.0', 'requests>=2.32.4', 'networkx', 'typing-extensions', + 'HLL', 'datasketches', 'packaging>=23.0', 'boto3>=1.37.15', + 'urllib3>=2.5.0'] # Pyupgrade - standardize and modernize Python syntax for newer versions of the language - repo: https://github.com/asottile/pyupgrade rev: v3.3.0 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..c7abf30eb --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +## Unreleased + +- Added compatibility support for NumPy 2.0 while constraining `numpy` to + `>=1.22.0,<3.0.0` to avoid future breakage from NumPy 3. +- Added compatibility support for Keras versions newer than 3.4.0 while + constraining `keras` to `>3.4.0,<4.0.0` to avoid future breakage from Keras 4. +- Updated the pre-commit configuration to align hook versions and hook + dependencies with the current project requirements. diff --git a/MANIFEST.in b/MANIFEST.in index bafea0779..f5ba88190 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ global-exclude .DS_Store global-exclude */__pycache__/* include *.txt +include CHANGELOG.md include CODEOWNERS recursive-include dataprofiler *.avro recursive-include dataprofiler *.csv diff --git a/dataprofiler/__init__.py b/dataprofiler/__init__.py index ed54be241..6b29c5b7a 100644 --- a/dataprofiler/__init__.py +++ b/dataprofiler/__init__.py @@ -1,4 +1,5 @@ """Package for dataprofiler.""" + from . import settings from ._version import get_versions from .data_readers.data import Data diff --git a/dataprofiler/_typing.py b/dataprofiler/_typing.py index fa362d1b1..b7a623881 100644 --- a/dataprofiler/_typing.py +++ b/dataprofiler/_typing.py @@ -1,4 +1,5 @@ """Contains typing aliases.""" + from typing import Dict, List, NewType, Union import numpy as np diff --git a/dataprofiler/data_readers/avro_data.py b/dataprofiler/data_readers/avro_data.py index 720b9d1f7..7f15bdec4 100644 --- a/dataprofiler/data_readers/avro_data.py +++ b/dataprofiler/data_readers/avro_data.py @@ -1,4 +1,5 @@ """Contains class for saving and loading spreadsheet data.""" + from io import BytesIO, StringIO from typing import Any, Dict, List, Optional, Union diff --git a/dataprofiler/data_readers/base_data.py b/dataprofiler/data_readers/base_data.py index 27d8d5deb..e6e85d3d4 100644 --- a/dataprofiler/data_readers/base_data.py +++ b/dataprofiler/data_readers/base_data.py @@ -1,4 +1,5 @@ """Contains abstract class for data loading and saving.""" + import locale import sys from collections import OrderedDict diff --git a/dataprofiler/data_readers/csv_data.py b/dataprofiler/data_readers/csv_data.py index 7e13d4075..cb1a2e2da 100644 --- a/dataprofiler/data_readers/csv_data.py +++ b/dataprofiler/data_readers/csv_data.py @@ -1,4 +1,5 @@ """Contains class that saves and loads spreadsheet data.""" + import csv import random import re diff --git a/dataprofiler/data_readers/data_utils.py b/dataprofiler/data_readers/data_utils.py index 611d25dc3..833d650ee 100644 --- a/dataprofiler/data_readers/data_utils.py +++ b/dataprofiler/data_readers/data_utils.py @@ -1,4 +1,5 @@ """Contains functions for data readers.""" + import json import logging import os @@ -334,7 +335,7 @@ def reservoir(file: TextIOWrapper, sample_nrows: int) -> list: except StopIteration: break # Append new, replace old with dummy, and keep track of order - remove_index = rng.integers(0, sample_nrows) + remove_index = int(rng.integers(0, sample_nrows)) values[indices[remove_index]] = str(None) indices[remove_index] = len(values) values.append(newval) diff --git a/dataprofiler/data_readers/filepath_or_buffer.py b/dataprofiler/data_readers/filepath_or_buffer.py index 56c21e28e..201f690eb 100644 --- a/dataprofiler/data_readers/filepath_or_buffer.py +++ b/dataprofiler/data_readers/filepath_or_buffer.py @@ -1,4 +1,5 @@ """Contains functions and classes for handling filepaths and buffers.""" + from io import BytesIO, StringIO, TextIOWrapper from typing import IO, Any, Optional, Type, Union, cast diff --git a/dataprofiler/data_readers/graph_data.py b/dataprofiler/data_readers/graph_data.py index 337408a68..3cc83b040 100644 --- a/dataprofiler/data_readers/graph_data.py +++ b/dataprofiler/data_readers/graph_data.py @@ -1,4 +1,5 @@ """Contains class for identifying, reading, and loading graph data.""" + import csv from typing import Dict, List, Optional, Union, cast diff --git a/dataprofiler/data_readers/json_data.py b/dataprofiler/data_readers/json_data.py index 93e5d7e6f..cc71c57e4 100644 --- a/dataprofiler/data_readers/json_data.py +++ b/dataprofiler/data_readers/json_data.py @@ -1,4 +1,5 @@ """Contains class to save and load json data.""" + import json import re import warnings @@ -71,9 +72,9 @@ def __init__( self._data_formats["records"] = self._get_data_as_records self._data_formats["json"] = self._get_data_as_json - self._data_formats[ - "flattened_dataframe" - ] = self._get_data_as_flattened_dataframe + self._data_formats["flattened_dataframe"] = ( + self._get_data_as_flattened_dataframe + ) self._selected_data_format: str = options.get( "data_format", "flattened_dataframe" ) diff --git a/dataprofiler/data_readers/parquet_data.py b/dataprofiler/data_readers/parquet_data.py index 4fa567b8d..b679431ba 100644 --- a/dataprofiler/data_readers/parquet_data.py +++ b/dataprofiler/data_readers/parquet_data.py @@ -1,4 +1,5 @@ """Contains class to save and load parquet data.""" + from io import BytesIO, StringIO from typing import Any, Dict, List, Optional, Union diff --git a/dataprofiler/data_readers/structured_mixins.py b/dataprofiler/data_readers/structured_mixins.py index 3587291f1..6b1da157d 100644 --- a/dataprofiler/data_readers/structured_mixins.py +++ b/dataprofiler/data_readers/structured_mixins.py @@ -1,4 +1,5 @@ """Contains mixin data class for loading datasets of tye SpreadSheet.""" + from logging import Logger from typing import Any, Dict, List, Optional, Union, cast diff --git a/dataprofiler/labelers/__init__.py b/dataprofiler/labelers/__init__.py index 1b2302fcf..a355ead23 100644 --- a/dataprofiler/labelers/__init__.py +++ b/dataprofiler/labelers/__init__.py @@ -26,6 +26,7 @@ 2. structured_model 3. regex_model """ + # import data labelers # import models from .base_data_labeler import BaseDataLabeler, TrainableDataLabeler diff --git a/dataprofiler/labelers/base_model.py b/dataprofiler/labelers/base_model.py index 032c2ea38..c5d7aef50 100644 --- a/dataprofiler/labelers/base_model.py +++ b/dataprofiler/labelers/base_model.py @@ -1,4 +1,5 @@ """Contains abstract classes for labeling data.""" + from __future__ import annotations import abc @@ -78,7 +79,7 @@ def __eq__(self, other: object) -> bool: :rtype: bool """ if ( - type(self) != type(other) + type(self) is not type(other) or not isinstance(other, BaseModel) or self._parameters != other._parameters or self._label_mapping != other._label_mapping diff --git a/dataprofiler/labelers/char_load_tf_model.py b/dataprofiler/labelers/char_load_tf_model.py index a4a44e03a..8adbfa0bd 100644 --- a/dataprofiler/labelers/char_load_tf_model.py +++ b/dataprofiler/labelers/char_load_tf_model.py @@ -1,4 +1,5 @@ """Contains class for training data labeler model.""" + from __future__ import annotations import copy @@ -17,6 +18,7 @@ from .. import dp_logging from . import labeler_utils from .base_model import AutoSubRegistrationMeta, BaseModel, BaseTrainableModel +from .character_level_cnn_model import ArgMaxLayer _file_dir = os.path.dirname(os.path.abspath(__file__)) @@ -29,6 +31,8 @@ class CharLoadTFModel(BaseTrainableModel, metaclass=AutoSubRegistrationMeta): # boolean if the label mapping requires the mapping for index 0 reserved requires_zero_mapping = False + _SOFTMAX_OUTPUT = "softmax_output" + _ARGMAX_OUTPUT = "argmax_output" def __init__( self, model_path: str, label_mapping: dict[str, int], parameters: dict = None @@ -61,6 +65,35 @@ def __init__( BaseModel.__init__(self, label_mapping, parameters) + @classmethod + def _create_model_outputs( + cls, softmax_output: tf.Tensor, argmax_output: tf.Tensor | None = None + ) -> dict[str, tf.Tensor]: + """Return normalized dict outputs for training and inference.""" + if argmax_output is None: + argmax_output = ArgMaxLayer(name=cls._ARGMAX_OUTPUT)(softmax_output) + return { + cls._SOFTMAX_OUTPUT: softmax_output, + cls._ARGMAX_OUTPUT: argmax_output, + } + + @classmethod + def _normalize_model_outputs(cls, model: tf.keras.Model) -> tf.keras.Model: + """Convert list-style outputs to the normalized dict structure.""" + return labeler_utils.normalize_tf_model_outputs( + model, + [cls._SOFTMAX_OUTPUT, cls._ARGMAX_OUTPUT], + lambda softmax_output, extra_outputs: cls._create_model_outputs( + softmax_output, extra_outputs[0] + ), + ) + + def _new_softmax_head_name(self) -> str: + """Return a layer name unique within the current model graph.""" + return labeler_utils.get_tf_rebuild_layer_name( + self._model, f"{self._SOFTMAX_OUTPUT}_rebuild" + ) + def __eq__(self, other: object) -> bool: """ Check if two models are equal with one another. @@ -215,15 +248,34 @@ def load_from_disk(cls, dirpath: str) -> CharLoadTFModel: tf_model = tf.keras.models.load_model(dirpath) loaded_model = cls(dirpath, label_mapping, parameters) - loaded_model._model = tf_model + loaded_model._model = cls._normalize_model_outputs(tf_model) # load self loaded_model._model_num_labels = loaded_model.num_labels loaded_model._model_default_ind = loaded_model.label_mapping[ loaded_model._parameters["default_label"] ] + loaded_model._compile_model(loaded_model.num_labels) return loaded_model + def _compile_model(self, num_labels: int) -> None: + """Compile the model with dict-based losses and metrics.""" + losses = { + self._SOFTMAX_OUTPUT: "categorical_crossentropy", + self._ARGMAX_OUTPUT: None, + } + f1_score_training = labeler_utils.F1Score( + num_classes=num_labels, average="micro" + ) + metrics = { + self._SOFTMAX_OUTPUT: [ + "categorical_crossentropy", + "acc", + f1_score_training, + ] + } + self._model.compile(loss=losses, optimizer="adam", metrics=metrics) + def _construct_model(self) -> None: """ Model constructor for the data labeler. @@ -237,46 +289,28 @@ def _construct_model(self) -> None: model_loc = self._parameters["model_path"] self._model: tf.keras.Model = tf.keras.models.load_model(model_loc) - self._model = tf.keras.Model(self._model.inputs, self._model.outputs) - softmax_output_layer_name = self._model.output_names[0] + self._model = self._normalize_model_outputs(self._model) + softmax_output = self._model.output[self._SOFTMAX_OUTPUT] + softmax_layer = softmax_output._keras_history[0] + softmax_output_layer_name = softmax_layer.name softmax_layer_ind = cast( int, labeler_utils.get_tf_layer_index_from_name( self._model, softmax_output_layer_name ), ) - softmax_layer = self._model.get_layer(softmax_output_layer_name) - new_softmax_layer = softmax_layer.output + new_softmax_layer = softmax_output if softmax_layer.weights[0].shape[-1] != num_labels: new_softmax_layer = tf.keras.layers.Dense( - num_labels, activation="softmax", name="softmax_output" + num_labels, + activation="softmax", + name=self._new_softmax_head_name(), )(self._model.layers[softmax_layer_ind - 1].output) - # Add argmax layer to get labels directly as an output - argmax_layer = tf.keras.ops.argmax(new_softmax_layer, axis=2) - - argmax_outputs = [new_softmax_layer, argmax_layer] - self._model = tf.keras.Model(self._model.inputs, argmax_outputs) - self._model = tf.keras.Model(self._model.inputs, self._model.outputs) - - # Compile the model w/ metrics - softmax_output_layer_name = self._model.output_names[0] - losses = {softmax_output_layer_name: "categorical_crossentropy"} - - # use f1 score metric - f1_score_training = labeler_utils.F1Score( - num_classes=num_labels, average="micro" - ) - metrics = { - softmax_output_layer_name: [ - "categorical_crossentropy", - "acc", - f1_score_training, - ] - } - - self._model.compile(loss=losses, optimizer="adam", metrics=metrics) + output_dict = self._create_model_outputs(new_softmax_layer) + self._model = tf.keras.Model(self._model.inputs, output_dict) + self._compile_model(num_labels) self._epoch_id = 0 self._model_num_labels = num_labels @@ -305,32 +339,14 @@ def _reconstruct_model(self) -> None: # Add the final Softmax layer to the previous spot # self._model.layers[-2] to skip: original softmax final_softmax_layer = tf.keras.layers.Dense( - num_labels, activation="softmax", name="softmax_output" + num_labels, + activation="softmax", + name=self._new_softmax_head_name(), )(self._model.layers[-2].output) - # Add argmax layer to get labels directly as an output - argmax_layer = tf.keras.ops.argmax(final_softmax_layer, axis=2) - - argmax_outputs = [final_softmax_layer, argmax_layer] - self._model = tf.keras.Model(self._model.inputs, argmax_outputs) - - # Compile the model - softmax_output_layer_name = self._model.output_names[0] - losses = {softmax_output_layer_name: "categorical_crossentropy"} - - # use f1 score metric - f1_score_training = labeler_utils.F1Score( - num_classes=num_labels, average="micro" - ) - metrics = { - softmax_output_layer_name: [ - "categorical_crossentropy", - "acc", - f1_score_training, - ] - } - - self._model.compile(loss=losses, optimizer="adam", metrics=metrics) + output_dict = self._create_model_outputs(final_softmax_layer) + self._model = tf.keras.Model(self._model.inputs, output_dict) + self._compile_model(num_labels) self._epoch_id = 0 self._model_num_labels = num_labels @@ -381,34 +397,53 @@ def fit( f1_report: dict = {} self._model.reset_metrics() - softmax_output_layer_name = self._model.output_names[0] start_time = time.time() batch_id = 0 for x_train, y_train in train_data: + y_train_dict = { + self._SOFTMAX_OUTPUT: y_train, + self._ARGMAX_OUTPUT: None, + } model_results = self._model.train_on_batch( - x_train, {softmax_output_layer_name: y_train} + x_train, + y_train_dict, + return_dict=True, + ) + acc_value = next( + (value for key, value in model_results.items() if key.endswith("acc")), + np.nan, + ) + f1_value = next( + (value for key, value in model_results.items() if "f1" in key.lower()), + np.nan, ) sys.stdout.flush() if verbose: sys.stdout.write( "\rEPOCH %d, batch_id %d: loss: %f - acc: %f - " - "f1_score %f" % (self._epoch_id, batch_id, *model_results[1:]) + "f1_score %f" + % ( + self._epoch_id, + batch_id, + model_results.get("loss", np.nan), + acc_value, + f1_value, + ) ) batch_id += 1 - for i, metric_label in enumerate(self._model.metrics_names): - history[metric_label] = model_results[i] + history.update(model_results) if val_data: f1, f1_report = self._validate_training(val_data) # type: ignore history["f1_report"] = f1_report - val_f1 = f1_report["weighted avg"]["f1-score"] if f1_report else np.NAN + val_f1 = f1_report["weighted avg"]["f1-score"] if f1_report else np.nan val_precision = ( - f1_report["weighted avg"]["precision"] if f1_report else np.NAN + f1_report["weighted avg"]["precision"] if f1_report else np.nan ) - val_recall = f1_report["weighted avg"]["recall"] if f1_report else np.NAN + val_recall = f1_report["weighted avg"]["recall"] if f1_report else np.nan epoch_time = time.time() - start_time logger.info( "\rEPOCH %d (%ds), loss: %f - acc: %f - f1_score %f -- " @@ -416,7 +451,9 @@ def fit( % ( self._epoch_id, epoch_time, - *model_results[1:], + model_results.get("loss", np.nan), + acc_value, + f1_value, val_f1, val_precision, val_recall, @@ -463,7 +500,7 @@ def _validate_training( y_val_pred.append( self._model.predict( x_val, batch_size=batch_size_test, verbose=verbose_keras - )[1] + )[self._ARGMAX_OUTPUT] ) y_val_test.append(np.argmax(y_val, axis=-1)) batch_id += 1 @@ -536,10 +573,10 @@ def predict( if show_confidences: confidences[ allocation_index : allocation_index + num_samples_in_batch - ] = model_output[0].numpy() - predictions[ - allocation_index : allocation_index + num_samples_in_batch - ] = model_output[1].numpy() + ] = model_output[self._SOFTMAX_OUTPUT].numpy() + predictions[allocation_index : allocation_index + num_samples_in_batch] = ( + model_output[self._ARGMAX_OUTPUT].numpy() + ) allocation_index += num_samples_in_batch diff --git a/dataprofiler/labelers/character_level_cnn_model.py b/dataprofiler/labelers/character_level_cnn_model.py index 2cbb7051a..7d78900af 100644 --- a/dataprofiler/labelers/character_level_cnn_model.py +++ b/dataprofiler/labelers/character_level_cnn_model.py @@ -1,4 +1,5 @@ """Contains classes for char data labeling.""" + from __future__ import annotations import copy @@ -120,6 +121,7 @@ def get_config(self): def call(self, argmax_layer: tf.Tensor, confidence_layer: tf.Tensor) -> tf.Tensor: """Apply the threshold argmax to the input tensor.""" + argmax_layer = tf.cast(argmax_layer, tf.int64) threshold_at_argmax = tf.gather(self.thresh_vec, argmax_layer) confidence_max_layer = tf.keras.backend.max(confidence_layer, axis=2) @@ -146,6 +148,19 @@ def call(self, argmax_layer: tf.Tensor, confidence_layer: tf.Tensor) -> tf.Tenso return final_predicted_layer +@tf.keras.utils.register_keras_serializable(package="CharacterLevelCnnModel") +class ArgMaxLayer(tf.keras.layers.Layer): + """Keras layer returning integer argmax indices.""" + + def call(self, confidence_layer: tf.Tensor) -> tf.Tensor: + """Return argmax indices as int64.""" + return tf.cast(tf.keras.ops.argmax(confidence_layer, axis=2), tf.int64) + + def compute_output_shape(self, input_shape): + """Return the confidence tensor shape without the class dimension.""" + return input_shape[:-1] + + @tf.keras.utils.register_keras_serializable(package="CharacterLevelCnnModel") class EncodingLayer(tf.keras.layers.Layer): """Encodes strings to integers.""" @@ -206,6 +221,9 @@ class CharacterLevelCnnModel(BaseTrainableModel, metaclass=AutoSubRegistrationMe # boolean if the label mapping requires the mapping for index 0 reserved requires_zero_mapping: bool = True + _SOFTMAX_OUTPUT = "softmax_output" + _ARGMAX_OUTPUT = "argmax_output" + _THRESH_OUTPUT = "thresh_argmax_output" def __init__(self, label_mapping: dict[str, int], parameters: dict = None) -> None: """ @@ -242,6 +260,54 @@ def __init__(self, label_mapping: dict[str, int], parameters: dict = None) -> No BaseModel.__init__(self, label_mapping, parameters) + @classmethod + def _create_model_outputs( + cls, + softmax_output: tf.Tensor, + default_ind: int, + num_labels: int, + argmax_output: tf.Tensor | None = None, + threshold_output: tf.Tensor | None = None, + ) -> dict[str, tf.Tensor]: + """Return normalized dict outputs for training and inference.""" + if argmax_output is None: + argmax_output = ArgMaxLayer(name=cls._ARGMAX_OUTPUT)(softmax_output) + if threshold_output is None: + threshold_output = ThreshArgMaxLayer( + threshold_=0.0, + num_labels_=num_labels, + default_ind=default_ind, + name=cls._THRESH_OUTPUT, + )(argmax_output, softmax_output) + return { + cls._SOFTMAX_OUTPUT: softmax_output, + cls._ARGMAX_OUTPUT: argmax_output, + cls._THRESH_OUTPUT: threshold_output, + } + + @classmethod + def _normalize_model_outputs( + cls, model: tf.keras.Model, default_ind: int, num_labels: int + ) -> tf.keras.Model: + """Convert list-style outputs to the normalized dict structure.""" + return labeler_utils.normalize_tf_model_outputs( + model, + [cls._SOFTMAX_OUTPUT, cls._ARGMAX_OUTPUT, cls._THRESH_OUTPUT], + lambda softmax_output, extra_outputs: cls._create_model_outputs( + softmax_output, + default_ind, + num_labels, + extra_outputs[0], + extra_outputs[1], + ), + ) + + def _new_softmax_head_name(self) -> str: + """Return a layer name unique within the current model graph.""" + return labeler_utils.get_tf_rebuild_layer_name( + self._model, f"{self._SOFTMAX_OUTPUT}_rebuild" + ) + def __eq__(self, other: object) -> bool: """ Check if two models are equal with one another. @@ -444,12 +510,19 @@ def load_from_disk(cls, dirpath: str) -> CharacterLevelCnnModel: loaded_model._construct_model() tf1_weights.append(loaded_model._model.weights[-1].value()) loaded_model._model.set_weights(tf1_weights) + else: + loaded_model._model = cls._normalize_model_outputs( + tf_model, + loaded_model.label_mapping[loaded_model._parameters["default_label"]], + loaded_model.num_labels, + ) # load self loaded_model._model_num_labels = loaded_model.num_labels loaded_model._model_default_ind = loaded_model.label_mapping[ loaded_model._parameters["default_label"] ] + loaded_model._compile_loss(loaded_model._model, loaded_model.num_labels) return loaded_model @staticmethod @@ -475,6 +548,26 @@ def _argmax_threshold_layer( # matrix. return ThreshArgMaxLayer(threshold, num_labels, default_ind) + @staticmethod + def _compile_loss(model: tf.keras.Model, num_labels: int) -> None: + """Compiles the loss for the given model and number of labels.""" + losses = { + CharacterLevelCnnModel._SOFTMAX_OUTPUT: "categorical_crossentropy", + CharacterLevelCnnModel._ARGMAX_OUTPUT: None, + CharacterLevelCnnModel._THRESH_OUTPUT: None, + } + f1_score_training = labeler_utils.F1Score( + num_classes=num_labels, average="micro" + ) + metrics = { + CharacterLevelCnnModel._SOFTMAX_OUTPUT: [ + "categorical_crossentropy", + "acc", + f1_score_training, + ] + } + model.compile(loss=losses, optimizer="adam", metrics=metrics) + def _construct_model(self) -> None: """ Construct model for the data labeler. @@ -555,39 +648,19 @@ def _construct_model(self) -> None: self._model.add(tf.keras.layers.Dropout(self._parameters["dropout"])) # Add the final Softmax layer - self._model.add(tf.keras.layers.Dense(num_labels, activation="softmax")) - - # Add argmax layer to get labels directly as an output - argmax_layer = tf.keras.ops.argmax(self._model.outputs[0], axis=2) - - # Create confidence layers - final_predicted_layer = ThreshArgMaxLayer( - threshold_=0.0, num_labels_=num_labels, default_ind=default_ind + self._model.add( + tf.keras.layers.Dense( + num_labels, + activation="softmax", + name=self._SOFTMAX_OUTPUT, + ) ) - argmax_outputs = self._model.outputs + [ - argmax_layer, - final_predicted_layer(argmax_layer, self._model.outputs[0]), - ] - self._model = tf.keras.Model(self._model.inputs, argmax_outputs) - - # Compile the model - softmax_output_layer_name = self._model.output_names[0] - losses = {softmax_output_layer_name: "categorical_crossentropy"} - - # use f1 score metric - f1_score_training = labeler_utils.F1Score( - num_classes=num_labels, average="micro" + output_dict = self._create_model_outputs( + self._model.outputs[0], default_ind, num_labels ) - metrics = { - softmax_output_layer_name: [ - "categorical_crossentropy", - "acc", - f1_score_training, - ] - } - - self._model.compile(loss=losses, optimizer="adam", metrics=metrics) + self._model = tf.keras.Model(self._model.inputs, output_dict) + self._compile_loss(self._model, num_labels) self._epoch_id = 0 self._model_num_labels = num_labels @@ -616,40 +689,16 @@ def _reconstruct_model(self) -> None: # Add the final Softmax layer to the previous spot # self._model.layers[-3] to skip: thresh and original softmax final_softmax_layer = tf.keras.layers.Dense( - num_labels, activation="softmax", name="dense_2" + num_labels, + activation="softmax", + name=self._new_softmax_head_name(), )(self._model.layers[-3].output) - # Add argmax layer to get labels directly as an output - argmax_layer = tf.keras.ops.argmax(final_softmax_layer, axis=2) - - # Create confidence layers - final_predicted_layer = ThreshArgMaxLayer( - threshold_=0.0, num_labels_=num_labels, default_ind=default_ind + output_dict = self._create_model_outputs( + final_softmax_layer, default_ind, num_labels ) - - argmax_outputs = [final_softmax_layer] + [ - argmax_layer, - final_predicted_layer(argmax_layer, final_softmax_layer), - ] - self._model = tf.keras.Model(self._model.inputs, argmax_outputs) - - # Compile the model - softmax_output_layer_name = self._model.output_names[0] - losses = {softmax_output_layer_name: "categorical_crossentropy"} - - # use f1 score metric - f1_score_training = labeler_utils.F1Score( - num_classes=num_labels, average="micro" - ) - metrics = { - softmax_output_layer_name: [ - "categorical_crossentropy", - "acc", - f1_score_training, - ] - } - - self._model.compile(loss=losses, optimizer="adam", metrics=metrics) + self._model = tf.keras.Model(self._model.inputs, output_dict) + self._compile_loss(self._model, num_labels) self._epoch_id = 0 self._model_num_labels = num_labels self._model_default_ind = default_ind @@ -699,34 +748,54 @@ def fit( f1_report: dict = {} self._model.reset_metrics() - softmax_output_layer_name = self._model.output_names[0] start_time = time.time() batch_id = 0 for x_train, y_train in train_data: + y_train_dict = { + self._SOFTMAX_OUTPUT: y_train, + self._ARGMAX_OUTPUT: None, + self._THRESH_OUTPUT: None, + } model_results = self._model.train_on_batch( - x_train, {softmax_output_layer_name: y_train} + x_train, + y_train_dict, + return_dict=True, + ) + acc_value = next( + (value for key, value in model_results.items() if key.endswith("acc")), + np.nan, + ) + f1_value = next( + (value for key, value in model_results.items() if "f1" in key.lower()), + np.nan, ) sys.stdout.flush() if verbose: sys.stdout.write( "\rEPOCH %d, batch_id %d: loss: %f - acc: %f - " - "f1_score %f" % (self._epoch_id, batch_id, *model_results[1:]) + "f1_score %f" + % ( + self._epoch_id, + batch_id, + model_results.get("loss", np.nan), + acc_value, + f1_value, + ) ) batch_id += 1 - for i, metric_label in enumerate(self._model.metrics_names): - history[metric_label] = model_results[i] + history.update(model_results) if val_data: f1, f1_report = self._validate_training(val_data) # type: ignore history["f1_report"] = f1_report - val_f1 = f1_report["weighted avg"]["f1-score"] if f1_report else np.NAN + val_f1 = f1_report["weighted avg"]["f1-score"] if f1_report else np.nan val_precision = ( - f1_report["weighted avg"]["precision"] if f1_report else np.NAN + f1_report["weighted avg"]["precision"] if f1_report else np.nan ) - val_recall = f1_report["weighted avg"]["recall"] if f1_report else np.NAN + val_recall = f1_report["weighted avg"]["recall"] if f1_report else np.nan epoch_time = time.time() - start_time logger.info( "\rEPOCH %d (%ds), loss: %f - acc: %f - f1_score %f -- " @@ -734,7 +803,9 @@ def fit( % ( self._epoch_id, epoch_time, - *model_results[1:], + model_results.get("loss", np.nan), + acc_value, + f1_value, val_f1, val_precision, val_recall, @@ -783,7 +854,7 @@ def _validate_training( tf.convert_to_tensor(x_val), batch_size=batch_size_test, verbose=verbose_keras, - )[1] + )[self._ARGMAX_OUTPUT] ) y_val_test.append(np.argmax(y_val, axis=-1)) batch_id += 1 @@ -876,10 +947,10 @@ def predict( if show_confidences: confidences[ allocation_index : allocation_index + num_samples_in_batch - ] = model_output[0].numpy() - predictions[ - allocation_index : allocation_index + num_samples_in_batch - ] = model_output[1].numpy() + ] = model_output[self._SOFTMAX_OUTPUT].numpy() + predictions[allocation_index : allocation_index + num_samples_in_batch] = ( + model_output[self._ARGMAX_OUTPUT].numpy() + ) sentence_lengths[ allocation_index : allocation_index + num_samples_in_batch ] = list(map(lambda x: len(x[0]), batch_data)) diff --git a/dataprofiler/labelers/classification_report_utils.py b/dataprofiler/labelers/classification_report_utils.py index 28e742e32..840c236fe 100644 --- a/dataprofiler/labelers/classification_report_utils.py +++ b/dataprofiler/labelers/classification_report_utils.py @@ -1,4 +1,5 @@ """Contains functions for classification.""" + from __future__ import annotations import warnings @@ -31,8 +32,8 @@ def convert_confusion_matrix_to_MCM(conf_matrix: list | np.ndarray) -> np.ndarra """ if not isinstance(conf_matrix, np.ndarray): conf_matrix = np.array(conf_matrix) - num_labels = conf_matrix.shape[0] - num_samples = np.sum(conf_matrix) + num_labels = len(conf_matrix) + num_samples: int = int(np.sum(conf_matrix)) MCM = np.zeros((num_labels, 2, 2), dtype=np.int64) # True Positives @@ -205,6 +206,8 @@ def precision_recall_fscore_support( f_score = (1 + beta2) * precision * recall / denom # Average the results + weights: np.ndarray | None + support: np.ndarray | None = true_sum if average == "weighted": weights = true_sum if weights.sum() == 0: @@ -219,9 +222,9 @@ def precision_recall_fscore_support( precision = np.average(precision, weights=weights) recall = np.average(recall, weights=weights) f_score = np.average(f_score, weights=weights) - true_sum = None # return no support + support = None # return no support - return precision, recall, f_score, true_sum + return precision, recall, f_score, support def classification_report( @@ -300,7 +303,7 @@ def classification_report( """ # ALTERATION: replaced the _check_targets with this if statement since # no y_true, y_pred - y_type = "multiclass" if conf_matrix.shape[0] > 2 else "binary" + y_type = "multiclass" if len(conf_matrix) > 2 else "binary" labels_given = True if labels is None: diff --git a/dataprofiler/labelers/column_name_model.py b/dataprofiler/labelers/column_name_model.py index d698cfd6a..1732983c0 100644 --- a/dataprofiler/labelers/column_name_model.py +++ b/dataprofiler/labelers/column_name_model.py @@ -1,4 +1,5 @@ """Contains class for column name data labeling model.""" + from __future__ import annotations import json diff --git a/dataprofiler/labelers/data_processing.py b/dataprofiler/labelers/data_processing.py index ba17a3bd3..70c980c39 100644 --- a/dataprofiler/labelers/data_processing.py +++ b/dataprofiler/labelers/data_processing.py @@ -73,7 +73,7 @@ def __eq__(self, other: object) -> bool: :rtype: bool """ if ( - type(self) != type(other) + type(self) is not type(other) or not isinstance(other, BaseDataProcessor) or self._parameters != other._parameters ): @@ -176,9 +176,11 @@ def process( labels: np.ndarray | None = None, label_mapping: dict[str, int] | None = None, batch_size: int = 32, - ) -> Generator[tuple[np.ndarray, np.ndarray] | np.ndarray, None, None] | tuple[ - np.ndarray, np.ndarray - ] | np.ndarray: + ) -> ( + Generator[tuple[np.ndarray, np.ndarray] | np.ndarray, None, None] + | tuple[np.ndarray, np.ndarray] + | np.ndarray + ): """Preprocess data.""" raise NotImplementedError() @@ -692,7 +694,7 @@ def process( :return batch_data: A dict containing samples of size batch_size :rtype batch_data: dicts """ - num_dim = sum([dim > 1 for dim in data.shape]) + num_dim = sum(dim > 1 for dim in np.shape(data)) if num_dim > 1: raise ValueError( "Multidimensional data given to " @@ -1213,10 +1215,10 @@ def match_sentence_lengths( :type inplace: bool :return: dict(pred=...) or dict(pred=..., conf=...) """ - pred_buffer = [] - conf_buffer = [] + pred_buffer: np.ndarray = np.array([]) + conf_buffer: np.ndarray = np.array([]) result_ind = 0 - buffer_add_inds = np.cumsum(list(map(len, results["pred"]))).tolist() + buffer_add_inds: list[int] = np.cumsum(list(map(len, results["pred"]))).tolist() separator_len = len(flatten_separator) if not inplace: @@ -1469,14 +1471,14 @@ def process( "If `labels` are specified, `label_mapping` " "must also be specified." ) - if data.shape != labels.shape: + if np.shape(data) != np.shape(labels): raise ValueError( f"Data and labels given to " f"StructCharPreprocessor are of different " - f"shapes, {data.shape} != {labels.shape}" + f"shapes, {np.shape(data)} != {np.shape(labels)}" ) - num_dim = sum([dim > 1 for dim in data.shape]) + num_dim = sum(dim > 1 for dim in np.shape(data)) if num_dim > 1: warnings.warn( "Data given to StructCharPreprocessor was " @@ -1589,7 +1591,7 @@ def __eq__(self, other: object) -> bool: :rtype: bool """ if ( - type(self) != type(other) + type(self) is not type(other) or not isinstance(other, StructCharPostprocessor) or self._parameters["default_label"] != other._parameters["default_label"] or self._parameters["pad_label"] != other._parameters["pad_label"] @@ -1681,10 +1683,10 @@ def match_sentence_lengths( :type inplace: bool :return: dict(pred=...) or dict(pred=..., conf=...) """ - pred_buffer = [] - conf_buffer = [] + pred_buffer: np.ndarray = np.array([]) + conf_buffer: np.ndarray = np.array([]) result_ind = 0 - buffer_add_inds = np.cumsum(list(map(len, results["pred"]))).tolist() + buffer_add_inds: list[int] = np.cumsum(list(map(len, results["pred"]))).tolist() separator_len = len(flatten_separator) if not inplace: @@ -1950,9 +1952,11 @@ def _validate_parameters(self, parameters: dict) -> None: # being changed and is already set aggregation_func = parameters.get( "aggregation_func", - self._parameters.get("aggregation_func") - if hasattr(self, "_parameters") - else None, + ( + self._parameters.get("aggregation_func") + if hasattr(self, "_parameters") + else None + ), ) if value is None and aggregation_func == "priority": errors.append( diff --git a/dataprofiler/labelers/labeler_utils.py b/dataprofiler/labelers/labeler_utils.py index 3a24886f3..3a1097ceb 100644 --- a/dataprofiler/labelers/labeler_utils.py +++ b/dataprofiler/labelers/labeler_utils.py @@ -1,4 +1,5 @@ """Contains functions for the data labeler.""" + from __future__ import annotations import logging @@ -78,8 +79,8 @@ class 1 1.00 0.67 0.80 3 def evaluate_accuracy( - predicted_entities_in_index: list[list[int]], - true_entities_in_index: list[list[int]], + predicted_entities_in_index: list[list[int]] | np.ndarray, + true_entities_in_index: list[list[int]] | np.ndarray, num_labels: int, entity_rev_dict: dict[int, str], verbose: bool = True, @@ -119,13 +120,16 @@ def evaluate_accuracy( if x[1] not in omitted_labels ] - max_len = len(predicted_entities_in_index[0]) - true_labels_padded = np.zeros((len(true_entities_in_index), max_len)) - for i, true_labels_row in enumerate(true_entities_in_index): + predicted_entities = [np.asarray(row) for row in predicted_entities_in_index] + true_entities = [np.asarray(row) for row in true_entities_in_index] + + max_len = len(predicted_entities[0]) + true_labels_padded = np.zeros((len(true_entities), max_len)) + for i, true_labels_row in enumerate(true_entities): true_labels_padded[i][: len(true_labels_row)] = true_labels_row true_labels_flatten = np.hstack(true_labels_padded) # type: ignore - predicted_labels_flatten = np.hstack(predicted_entities_in_index) + predicted_labels_flatten = np.hstack(predicted_entities) all_labels: list[str] = [] if entity_rev_dict: @@ -228,6 +232,64 @@ def get_tf_layer_index_from_name(model: tf.keras.Model, layer_name: str) -> int return None +def normalize_tf_model_outputs( + model: tf.keras.Model, + output_names: list[str], + create_outputs_fn: Callable[ + [tf.Tensor, list[tf.Tensor | None]], dict[str, tf.Tensor] + ], +) -> tf.keras.Model: + """Convert a model's outputs into a named dict-output structure when possible.""" + try: + model_output = model.output + except (AttributeError, IndexError): + model_output = None + + try: + model_outputs_list = list(model.outputs) + except (AttributeError, IndexError, TypeError): + model_outputs_list = [] + + if isinstance(model_output, dict): + if set(model_output) == set(output_names): + return model + softmax_output = model_output.get( + output_names[0], next(iter(model_output.values())) + ) + extra_outputs = [model_output.get(name) for name in output_names[1:]] + else: + if not model_outputs_list: + try: + last_output = model.layers[-1].output + except (AttributeError, IndexError): + return model + if not hasattr(last_output, "_keras_history"): + return model + model_outputs_list = [last_output] + softmax_output = model_outputs_list[0] + extra_outputs = [ + model_outputs_list[index] if len(model_outputs_list) > index else None + for index in range(1, len(output_names)) + ] + + try: + output_dict = create_outputs_fn(softmax_output, extra_outputs) + return tf.keras.Model(model.inputs, output_dict) + except (AttributeError, TypeError, ValueError): + return model + + +def get_tf_rebuild_layer_name(model: tf.keras.Model, base_name: str) -> str: + """Return a layer name unique within the current model graph.""" + existing_names = {layer.name for layer in getattr(model, "layers", [])} + if base_name not in existing_names: + return base_name + suffix = 1 + while f"{base_name}_{suffix}" in existing_names: + suffix += 1 + return f"{base_name}_{suffix}" + + def hide_tf_logger_warnings() -> None: """Filter out a set of warnings from the tf logger.""" diff --git a/dataprofiler/labelers/regex_model.py b/dataprofiler/labelers/regex_model.py index c6a690c17..dd74da71a 100644 --- a/dataprofiler/labelers/regex_model.py +++ b/dataprofiler/labelers/regex_model.py @@ -1,4 +1,5 @@ """Contains class for regex data labeling model.""" + from __future__ import annotations import copy diff --git a/dataprofiler/plugins/decorators.py b/dataprofiler/plugins/decorators.py index c781f4300..f099c1aa4 100644 --- a/dataprofiler/plugins/decorators.py +++ b/dataprofiler/plugins/decorators.py @@ -1,4 +1,5 @@ """Contains function for generating plugins data.""" + from collections import defaultdict from typing import Any, DefaultDict, Dict @@ -21,7 +22,6 @@ def __inner_factory_function(fn): :param fn: Plugin function :return: function """ - global plugins_dict plugins_dict[typ][name] = fn return fn diff --git a/dataprofiler/profilers/__init__.py b/dataprofiler/profilers/__init__.py index 4b068fcb0..14834794b 100644 --- a/dataprofiler/profilers/__init__.py +++ b/dataprofiler/profilers/__init__.py @@ -1,4 +1,5 @@ """Package for providing statistics and predictions for a given dataset.""" + from . import json_decoder from .base_column_profilers import BaseColumnProfiler from .categorical_column_profile import CategoricalColumn diff --git a/dataprofiler/profilers/categorical_column_profile.py b/dataprofiler/profilers/categorical_column_profile.py index 1ca630900..d64f5aa5b 100644 --- a/dataprofiler/profilers/categorical_column_profile.py +++ b/dataprofiler/profilers/categorical_column_profile.py @@ -1,4 +1,5 @@ """Contains class for categorical column profiler.""" + from __future__ import annotations import math @@ -277,28 +278,28 @@ def diff(self, other_profile: CategoricalColumn, options: dict = None) -> dict: # These stats are only diffed if both profiles are categorical if self.is_match and other_profile.is_match: - differences["statistics"][ - "chi2-test" - ] = profiler_utils.perform_chi_squared_test_for_homogeneity( - self._categories, - self.sample_size, - other_profile._categories, - other_profile.sample_size, + differences["statistics"]["chi2-test"] = ( + profiler_utils.perform_chi_squared_test_for_homogeneity( + self._categories, + self.sample_size, + other_profile._categories, + other_profile.sample_size, + ) ) - differences["statistics"][ - "categories" - ] = profiler_utils.find_diff_of_lists_and_sets( - self.categories, other_profile.categories + differences["statistics"]["categories"] = ( + profiler_utils.find_diff_of_lists_and_sets( + self.categories, other_profile.categories + ) ) - differences["statistics"][ - "gini_impurity" - ] = profiler_utils.find_diff_of_numbers( - self.gini_impurity, other_profile.gini_impurity + differences["statistics"]["gini_impurity"] = ( + profiler_utils.find_diff_of_numbers( + self.gini_impurity, other_profile.gini_impurity + ) ) - differences["statistics"][ - "unalikeability" - ] = profiler_utils.find_diff_of_numbers( - self.unalikeability, other_profile.unalikeability + differences["statistics"]["unalikeability"] = ( + profiler_utils.find_diff_of_numbers( + self.unalikeability, other_profile.unalikeability + ) ) cat_count1 = dict( sorted(self._categories.items(), key=itemgetter(1), reverse=True) @@ -326,9 +327,9 @@ def diff(self, other_profile: CategoricalColumn, options: dict = None) -> dict: ) differences["statistics"]["psi"] = total_psi - differences["statistics"][ - "categorical_count" - ] = profiler_utils.find_diff_of_dicts(self_cat_count, other_cat_count) + differences["statistics"]["categorical_count"] = ( + profiler_utils.find_diff_of_dicts(self_cat_count, other_cat_count) + ) return differences diff --git a/dataprofiler/profilers/column_profile_compilers.py b/dataprofiler/profilers/column_profile_compilers.py index 07edf13dc..cfeb8c699 100644 --- a/dataprofiler/profilers/column_profile_compilers.py +++ b/dataprofiler/profilers/column_profile_compilers.py @@ -1,4 +1,5 @@ """For generating a report.""" + from __future__ import annotations import abc diff --git a/dataprofiler/profilers/data_labeler_column_profile.py b/dataprofiler/profilers/data_labeler_column_profile.py index d9bfe1ee9..3ce6257f9 100644 --- a/dataprofiler/profilers/data_labeler_column_profile.py +++ b/dataprofiler/profilers/data_labeler_column_profile.py @@ -1,4 +1,5 @@ """Contains class for for profiling data labeler col.""" + from __future__ import annotations import operator @@ -427,7 +428,7 @@ def _update_predictions( start_index = 0 if self.data_labeler.model.requires_zero_mapping: start_index = 1 - for i in range(rank_predictions.shape[0]): + for i in range(len(rank_predictions)): sorted_rank = rank_predictions[i][-self._top_k_voting :] sorted_rank = sorted_rank[np.argsort(predictions["conf"][i][sorted_rank])] for rank_position, value in enumerate(sorted_rank): diff --git a/dataprofiler/profilers/datetime_column_profile.py b/dataprofiler/profilers/datetime_column_profile.py index af99283a9..1042ea0c0 100644 --- a/dataprofiler/profilers/datetime_column_profile.py +++ b/dataprofiler/profilers/datetime_column_profile.py @@ -1,4 +1,5 @@ """Contains class for profiling datetime column.""" + from __future__ import annotations import datetime @@ -216,7 +217,7 @@ def _validate_datetime(date: str, date_format: str) -> datetime.datetime | float :return: either the str converted into a date format, or Nan """ try: - converted_date: (datetime.datetime | float) = datetime.datetime.strptime( + converted_date: datetime.datetime | float = datetime.datetime.strptime( date, date_format ) except (ValueError, TypeError): @@ -237,7 +238,7 @@ def _replace_day_suffix(date: str, pattern: re.Pattern) -> str | float: """ try: new_date: str | float = pattern.sub(r"\1", date) - except (TypeError): + except TypeError: new_date = np.nan return new_date diff --git a/dataprofiler/profilers/float_column_profile.py b/dataprofiler/profilers/float_column_profile.py index bc426a447..3d6ede326 100644 --- a/dataprofiler/profilers/float_column_profile.py +++ b/dataprofiler/profilers/float_column_profile.py @@ -1,4 +1,5 @@ """Float profile analysis for individual col within structured profiling.""" + from __future__ import annotations import copy diff --git a/dataprofiler/profilers/graph_profiler.py b/dataprofiler/profilers/graph_profiler.py index 0680a29a7..345a0f2e5 100644 --- a/dataprofiler/profilers/graph_profiler.py +++ b/dataprofiler/profilers/graph_profiler.py @@ -1,4 +1,5 @@ """Class and functions to calculate and profile properties of graph data.""" + from __future__ import annotations import importlib diff --git a/dataprofiler/profilers/helpers/__init__.py b/dataprofiler/profilers/helpers/__init__.py index 43393433d..2c72b2f35 100644 --- a/dataprofiler/profilers/helpers/__init__.py +++ b/dataprofiler/profilers/helpers/__init__.py @@ -1,4 +1,5 @@ """This package provides helper functions for generating reports.""" + from .report_helpers import _prepare_report, calculate_quantiles __all__ = [ diff --git a/dataprofiler/profilers/helpers/report_helpers.py b/dataprofiler/profilers/helpers/report_helpers.py index 0588252c9..44ac8fb1a 100644 --- a/dataprofiler/profilers/helpers/report_helpers.py +++ b/dataprofiler/profilers/helpers/report_helpers.py @@ -1,4 +1,5 @@ """Contains helper functions for generating report.""" + from __future__ import annotations import math diff --git a/dataprofiler/profilers/histogram_utils.py b/dataprofiler/profilers/histogram_utils.py index df230c4c7..b7aed9ad1 100644 --- a/dataprofiler/profilers/histogram_utils.py +++ b/dataprofiler/profilers/histogram_utils.py @@ -7,15 +7,33 @@ A copy of the license for numpy is available here: https://github.com/numpy/numpy/blob/main/LICENSE.txt """ + import operator from typing import List, Optional, Tuple, Union import numpy as np -from numpy.lib.histograms import ( # type: ignore[attr-defined] - _get_outer_edges, - _hist_bin_selectors, - _unsigned_subtract, -) + +from . import profiler_utils + +try: + # numpy v2+ + # NOTE: `numpy.lib._histograms_impl` is a private module, so this import may + # need to be revisited if NumPy exposes a public replacement for these + # helpers. NumPy's 2.4.0 release notes discuss public APIs replacing modules + # that moved private in 2.x: https://numpy.org/doc/stable/release/2.4.0-notes.html + from numpy.lib._histograms_impl import ( # type: ignore[attr-defined] + _get_outer_edges, + _hist_bin_selectors, + _unsigned_subtract, + ) + +except ModuleNotFoundError: + # numpy v1+ + from numpy.lib.histograms import ( + _get_outer_edges, + _hist_bin_selectors, + _unsigned_subtract, + ) def _get_maximum_from_profile(profile): @@ -79,7 +97,7 @@ def _ptp(maximum: float, minimum: float): :return: the difference between the maximum and minimum """ - return np.subtract(maximum, minimum) + return profiler_utils.as_float_scalar(np.subtract(maximum, minimum)) def _calc_doane_bin_width_from_profile(profile): @@ -180,7 +198,9 @@ def _calc_fd_bin_width_from_profile(profile): :return: An estimate of the optimal bin width for the given data. """ - iqr = np.subtract(profile._get_percentile([75]), profile._get_percentile([25])) + iqr = profiler_utils.as_float_scalar( + np.subtract(profile._get_percentile([75]), profile._get_percentile([25])) + ) dataset_size = _get_dataset_size_from_profile(profile) return 2.0 * iqr * dataset_size ** (-1.0 / 3.0) @@ -290,6 +310,8 @@ def _get_bin_edges( else: # Do not call selectors on empty arrays width = _hist_bin_selectors[bin_name](a, (first_edge, last_edge)) + if width is not None: + width = profiler_utils.as_float_scalar(width) if width: n_equal_bins = int( np.ceil(_unsigned_subtract(last_edge, first_edge) / width) @@ -341,6 +363,8 @@ def _calculate_bins_from_profile(profile, bin_method): else: # Do not call selectors on empty arrays width = _hist_bin_width_selectors_for_profile[bin_method](profile) + if width is not None: + width = profiler_utils.as_float_scalar(width) if width and not np.isnan(width): n_equal_bins = int(np.ceil(_ptp(maximum, minimum) / width)) else: diff --git a/dataprofiler/profilers/int_column_profile.py b/dataprofiler/profilers/int_column_profile.py index 014465c71..ae4ed5759 100644 --- a/dataprofiler/profilers/int_column_profile.py +++ b/dataprofiler/profilers/int_column_profile.py @@ -1,4 +1,5 @@ """Int profile analysis for individual col within structured profiling.""" + from __future__ import annotations import numpy as np @@ -163,7 +164,7 @@ def update(self, df_series: pd.Series) -> IntColumn: df_series = df_series.reset_index(drop=True) is_each_row_int = self._is_each_row_int(df_series) sample_size = len(is_each_row_int) - match_int_count = np.sum(is_each_row_int) + match_int_count: int = int(np.sum(is_each_row_int)) profile = dict(match_count=match_int_count, sample_size=sample_size) BaseColumnProfiler._perform_property_calcs( diff --git a/dataprofiler/profilers/json_decoder.py b/dataprofiler/profilers/json_decoder.py index fb4ff8cb9..eb09db0d7 100644 --- a/dataprofiler/profilers/json_decoder.py +++ b/dataprofiler/profilers/json_decoder.py @@ -1,4 +1,5 @@ """Contains methods to decode components of a Profiler.""" + from __future__ import annotations import warnings @@ -116,9 +117,9 @@ def get_structured_col_profiler_class(class_name: str) -> type[StructuredColProf :type class_name: str representing name of class :return: subclass of StructuredColProfiler object """ - struct_col_profiler_class: None | ( - type[StructuredColProfiler] - ) = _structured_col_profiler.get(class_name) + struct_col_profiler_class: None | (type[StructuredColProfiler]) = ( + _structured_col_profiler.get(class_name) + ) if struct_col_profiler_class is None: raise ValueError( f"Invalid structured col profiler class {class_name} " f"failed to load." @@ -153,9 +154,9 @@ def load_column_profile( JSON """ - column_profiler_cls: type[ - BaseColumnProfiler[BaseColumnProfiler] - ] = get_column_profiler_class(serialized_json["class"]) + column_profiler_cls: type[BaseColumnProfiler[BaseColumnProfiler]] = ( + get_column_profiler_class(serialized_json["class"]) + ) return column_profiler_cls.load_from_dict(serialized_json["data"], config) diff --git a/dataprofiler/profilers/numerical_column_stats.py b/dataprofiler/profilers/numerical_column_stats.py index fa0666a66..9ec2190a3 100644 --- a/dataprofiler/profilers/numerical_column_stats.py +++ b/dataprofiler/profilers/numerical_column_stats.py @@ -6,7 +6,7 @@ import copy import itertools import warnings -from typing import Any, Callable, Dict, List, TypeVar, cast +from typing import Any, Callable, Dict, List, TypeAlias, TypeVar, cast import numpy as np import numpy.typing as npt @@ -32,6 +32,7 @@ def __init__(self, function: Callable) -> None: NumericStatsMixinT = TypeVar("NumericStatsMixinT", bound="NumericStatsMixin") +Numeric: TypeAlias = int | float | np.float64 | np.int64 class NumericStatsMixin(BaseColumnProfiler[NumericStatsMixinT], metaclass=abc.ABCMeta): @@ -56,10 +57,10 @@ def __init__(self, options: NumericalOptions = None) -> None: "NumericalStatsMixin parameter 'options' must be " "of type NumericalOptions." ) - self.min: int | float | np.float64 | np.int64 | None = None - self.max: int | float | np.float64 | np.int64 | None = None + self.min: Numeric | None = None + self.max: Numeric | None = None self._top_k_modes: int = 5 # By default, return at max 5 modes - self.sum: int | float | np.float64 | np.int64 = np.float64(0) + self.sum: Numeric = np.float64(0) self._biased_variance: float | np.float64 = np.nan self._biased_skewness: float | np.float64 = np.nan self._biased_kurtosis: float | np.float64 = np.nan @@ -198,7 +199,7 @@ def _add_helper_merge_profile_histograms( # calculate the min of the first edge and the max of the last edge # between two arrays global_min_of_histogram_edges = ( - float(self.min) + profiler_utils.as_float_scalar(self.min) if self.min is not None else min( other1._stored_histogram["histogram"]["bin_edges"][0], @@ -207,7 +208,7 @@ def _add_helper_merge_profile_histograms( ) global_max_of_histogram_edges = ( - float(self.max) + profiler_utils.as_float_scalar(self.max) if self.max is not None else max( other1._stored_histogram["histogram"]["bin_edges"][-1], @@ -257,9 +258,9 @@ def _add_helper_merge_profile_histograms( if self.user_set_histogram_bin is None: for method in self.histogram_bin_method_names: - self.histogram_methods[method][ - "suggested_bin_count" - ] = histogram_utils._calculate_bins_from_profile(self, method) + self.histogram_methods[method]["suggested_bin_count"] = ( + histogram_utils._calculate_bins_from_profile(self, method) + ) self._get_quantiles() @@ -298,14 +299,14 @@ def _add_helper( ) if "min" in self.__calculations.keys(): if other1.min is not None and other2.min is not None: - self.min = min(other1.min, other2.min) + self.min = min(other1.min, other2.min) # type: ignore[type-var] elif other2.min is None: self.min = other1.min else: self.min = other2.min if "max" in self.__calculations.keys(): if other1.max is not None and other2.max is not None: - self.max = max(other1.max, other2.max) + self.max = max(other1.max, other2.max) # type: ignore[type-var] elif other2.max is None: self.max = other1.max else: @@ -540,7 +541,7 @@ def mean(self) -> float | np.float64: """Return mean value.""" if self.match_count == 0: return 0.0 - return self.sum / self.match_count + return cast(float | np.float64, self.sum / self.match_count) @property def mode(self) -> list[float]: @@ -622,14 +623,17 @@ def _perform_t_test( RuntimeWarning, ) invalid_stats = True - if np.isnan( - [float(mean1), float(mean2), float(var1), float(var2)] - ).any() or None in [ - mean1, - mean2, - var1, - var2, - ]: + if ( + None in [mean1, mean2, var1, var2] + or np.isnan( + [ + profiler_utils.as_float_scalar(mean1), + profiler_utils.as_float_scalar(mean2), + profiler_utils.as_float_scalar(var1), + profiler_utils.as_float_scalar(var2), + ] + ).any() + ): warnings.warn( "Null value(s) found in mean and/or variance values. " "T-test cannot be performed.", @@ -1040,10 +1044,7 @@ def _merge_biased_kurtosis( / N**3 ) third_term = ( - 6 - * delta**2 - * (match_count1**2 * M2_2 + match_count2**2 * M2_1) - / N**2 + 6 * delta**2 * (match_count1**2 * M2_2 + match_count2**2 * M2_1) / N**2 ) fourth_term = 4 * delta * (match_count1 * M3_2 - match_count2 * M3_1) / N M4 = first_term + second_term + third_term + fourth_term @@ -1403,7 +1404,7 @@ def _assimilate_histogram( dest_hist_entity_count_per_bin: np.ndarray, dest_hist_bin_edges: np.ndarray, dest_hist_num_bin: int, - ) -> tuple[dict[str, np.ndarray[Any, Any]], float]: + ) -> tuple[dict[str, np.ndarray], float]: """ Assimilates a histogram into another histogram using specifications. @@ -1821,11 +1822,15 @@ def _get_variance( # Suppress any numpy warnings as we have a custom warning for invalid # or infinite data already with np.errstate(all="ignore"): - batch_biased_variance = np.var(df_series) # Obtains biased variance + batch_biased_variance = cast(float | np.float64, np.var(df_series)) subset_properties["biased_variance"] = batch_biased_variance sum_value = subset_properties["sum"] batch_count = subset_properties["match_count"] - batch_mean = 0.0 if not batch_count else float(sum_value) / batch_count + batch_mean = ( + 0.0 + if not batch_count + else profiler_utils.as_float_scalar(sum_value) / batch_count + ) subset_properties["mean"] = batch_mean self._biased_variance = self._merge_biased_variance( self.match_count, diff --git a/dataprofiler/profilers/order_column_profile.py b/dataprofiler/profilers/order_column_profile.py index 308262324..0a4374312 100644 --- a/dataprofiler/profilers/order_column_profile.py +++ b/dataprofiler/profilers/order_column_profile.py @@ -1,4 +1,5 @@ """Index profile analysis for individual col within structured profiling.""" + from __future__ import annotations from abc import abstractmethod diff --git a/dataprofiler/profilers/profile_builder.py b/dataprofiler/profilers/profile_builder.py index 6e512658f..7d904b6a8 100644 --- a/dataprofiler/profilers/profile_builder.py +++ b/dataprofiler/profilers/profile_builder.py @@ -1919,10 +1919,10 @@ def diff( # type: ignore[override] col_name = other_profile._profile[i].name other_profile_schema[col_name].append(i) - report["global_stats"][ - "profile_schema" - ] = profiler_utils.find_diff_of_dicts_with_diff_keys( - self_profile_schema, other_profile_schema + report["global_stats"]["profile_schema"] = ( + profiler_utils.find_diff_of_dicts_with_diff_keys( + self_profile_schema, other_profile_schema + ) ) # Only find the diff of columns if the schemas are exactly the same @@ -2101,9 +2101,9 @@ def report(self, report_options: dict = None) -> dict: self.options.null_replication_metrics.is_enabled and i in self._null_replication_metrics ): - report["data_stats"][i][ - "null_replication_metrics" - ] = self._null_replication_metrics[i] + report["data_stats"][i]["null_replication_metrics"] = ( + self._null_replication_metrics[i] + ) return _prepare_report(report, output_format, omit_keys) @@ -2610,9 +2610,11 @@ def _update_null_replication_metrics(self, clean_samples: dict) -> None: total_row_sum = np.asarray( [ - get_data_type_profiler(profile).sum - if get_data_type(profile) not in [None, "datetime"] - else np.nan + ( + get_data_type_profiler(profile).sum + if get_data_type(profile) not in [None, "datetime"] + else np.nan + ) for profile in self._profile ] ) @@ -2704,17 +2706,21 @@ def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: self_row_sum = np.asarray( [ - get_data_type_profiler(profile).sum - if get_data_type(profile) - else np.nan + ( + get_data_type_profiler(profile).sum + if get_data_type(profile) + else np.nan + ) for profile in self._profile ] ) other_row_sum = np.asarray( [ - get_data_type_profiler(profile).sum - if get_data_type(profile) - else np.nan + ( + get_data_type_profiler(profile).sum + if get_data_type(profile) + else np.nan + ) for profile in other._profile ] ) diff --git a/dataprofiler/profilers/profiler_utils.py b/dataprofiler/profilers/profiler_utils.py index e38e1b041..7986cec07 100644 --- a/dataprofiler/profilers/profiler_utils.py +++ b/dataprofiler/profilers/profiler_utils.py @@ -1,4 +1,5 @@ """Contains functions for profilers.""" + from __future__ import annotations import collections @@ -39,6 +40,18 @@ from .. import rng_utils +def as_float_scalar( + value: int | float | np.integer | np.floating | np.ndarray | list[float], +) -> float: + """Convert a scalar-like value to a Python float.""" + array_value = np.asarray(value) + if array_value.ndim == 0: + return float(array_value) + if array_value.size == 1: + return float(array_value.item()) + raise TypeError("Expected a scalar-like numeric value.") + + def recursive_dict_update(d: dict, update_d: dict) -> dict: """ Recursive updates nested dictionaries. Updating d with update_d. @@ -417,13 +430,11 @@ def __sub__(self: T, other: T) -> Any: def find_diff_of_numbers( stat1: int | float | np.float64 | np.int64 | None, stat2: int | float | np.float64 | np.int64 | None, -) -> Any: - ... +) -> Any: ... @overload -def find_diff_of_numbers(stat1: T | None, stat2: T | None) -> Any: - ... +def find_diff_of_numbers(stat1: T | None, stat2: T | None) -> Any: ... def find_diff_of_numbers(stat1, stat2): @@ -602,7 +613,7 @@ def find_diff_of_matrices( mat1 = np.array(matrix1, dtype=np.float64) mat2 = np.array(matrix2, dtype=np.float64) - if mat1.shape == mat2.shape: + if np.shape(mat1) == np.shape(mat2): diff: np.ndarray = mat1 - mat2 if ((diff == 0) | np.isnan(diff)).all(): return "unchanged" diff --git a/dataprofiler/profilers/text_column_profile.py b/dataprofiler/profilers/text_column_profile.py index bea8dbd68..eb79643fd 100644 --- a/dataprofiler/profilers/text_column_profile.py +++ b/dataprofiler/profilers/text_column_profile.py @@ -1,4 +1,5 @@ """Text profile analysis for individual col within structured profiling..""" + from __future__ import annotations import itertools diff --git a/dataprofiler/profilers/unstructured_labeler_profile.py b/dataprofiler/profilers/unstructured_labeler_profile.py index 1c7b16c0f..22789c4e3 100644 --- a/dataprofiler/profilers/unstructured_labeler_profile.py +++ b/dataprofiler/profilers/unstructured_labeler_profile.py @@ -1,4 +1,5 @@ """Profile analysis for applying labels within unstructured profiling.""" + from __future__ import annotations from collections import defaultdict diff --git a/dataprofiler/profilers/unstructured_text_profile.py b/dataprofiler/profilers/unstructured_text_profile.py index 96b7d0625..3f1b6dd7f 100644 --- a/dataprofiler/profilers/unstructured_text_profile.py +++ b/dataprofiler/profilers/unstructured_text_profile.py @@ -1,4 +1,5 @@ """For profiling unstructured text data.""" + from __future__ import annotations import itertools diff --git a/dataprofiler/reports/graphs.py b/dataprofiler/reports/graphs.py index 1f0b43018..4e630a1c5 100644 --- a/dataprofiler/reports/graphs.py +++ b/dataprofiler/reports/graphs.py @@ -1,4 +1,5 @@ """Contains functions for generating graph data report.""" + # !/usr/bin/env python3 from __future__ import annotations diff --git a/dataprofiler/reports/utils.py b/dataprofiler/reports/utils.py index a10b8fe59..975dc7d88 100644 --- a/dataprofiler/reports/utils.py +++ b/dataprofiler/reports/utils.py @@ -1,4 +1,5 @@ """Contains functions for checking for installations/dependencies.""" + import sys import warnings from typing import Any, Callable, List, TypeVar, cast diff --git a/dataprofiler/rng_utils.py b/dataprofiler/rng_utils.py index 329066658..2fd14f0f7 100644 --- a/dataprofiler/rng_utils.py +++ b/dataprofiler/rng_utils.py @@ -1,4 +1,5 @@ """Create a random number generator using a manual seed DATAPROFILER_SEED.""" + import os import warnings diff --git a/dataprofiler/settings.py b/dataprofiler/settings.py index 1ba017f4e..a81c3477c 100644 --- a/dataprofiler/settings.py +++ b/dataprofiler/settings.py @@ -1,2 +1,3 @@ """Configure settings for dataprofiler.""" + _seed = None diff --git a/dataprofiler/tests/labelers/test_char_tf_load_model.py b/dataprofiler/tests/labelers/test_char_tf_load_model.py index 40879e579..6160b8fa4 100644 --- a/dataprofiler/tests/labelers/test_char_tf_load_model.py +++ b/dataprofiler/tests/labelers/test_char_tf_load_model.py @@ -227,6 +227,26 @@ def test_predict(self, *mocks): self.assertIn("conf", result) self.assertEqual((2, 2, model.num_labels), np.array(result["conf"]).shape) + def test_normalize_old_list_output_model(self, *mocks): + inputs = tf.keras.Input(shape=(2,), dtype=tf.int64) + embedded = tf.keras.layers.Embedding(input_dim=100, output_dim=8)(inputs) + softmax_output = tf.keras.layers.Dense( + self.label_mapping["ADDRESS"] + 1, + activation="softmax", + )(embedded) + argmax_output = tf.keras.layers.Lambda( + lambda x: tf.cast(tf.argmax(x, axis=2), tf.int64) + )(softmax_output) + old_format_model = tf.keras.Model(inputs, [softmax_output, argmax_output]) + + normalized_model = CharLoadTFModel._normalize_model_outputs(old_format_model) + + self.assertIsInstance(normalized_model.output, dict) + self.assertSetEqual( + set(normalized_model.output.keys()), + {CharLoadTFModel._SOFTMAX_OUTPUT, CharLoadTFModel._ARGMAX_OUTPUT}, + ) + def test_fit_and_predict(self, *mocks): # model model = CharLoadTFModel(self.model_path, self.label_mapping) diff --git a/dataprofiler/tests/labelers/test_character_level_cnn_model.py b/dataprofiler/tests/labelers/test_character_level_cnn_model.py index cbc35b131..311da0066 100644 --- a/dataprofiler/tests/labelers/test_character_level_cnn_model.py +++ b/dataprofiler/tests/labelers/test_character_level_cnn_model.py @@ -10,8 +10,10 @@ from dataprofiler.labelers import utils as labeler_utils from dataprofiler.labelers.character_level_cnn_model import ( + ArgMaxLayer, CharacterLevelCnnModel, EncodingLayer, + ThreshArgMaxLayer, ) _file_dir = os.path.dirname(os.path.abspath(__file__)) @@ -253,6 +255,36 @@ def test_validation_evaluate_and_classification_report(self, *mocks): self.assertIsNotNone(f1_report) self.assertEqual(11, f1_report["ADDRESS"]["support"]) + def test_normalize_old_list_output_model(self): + default_ind = self.label_mapping["UNKNOWN"] + num_labels = max(self.label_mapping.values()) + 1 + inputs = tf.keras.Input(shape=(2, 4)) + hidden = tf.keras.layers.Dense(8, activation="relu")(inputs) + softmax_output = tf.keras.layers.Dense(num_labels, activation="softmax")(hidden) + argmax_output = ArgMaxLayer()(softmax_output) + threshold_output = ThreshArgMaxLayer( + threshold_=0.0, + num_labels_=num_labels, + default_ind=default_ind, + )(argmax_output, softmax_output) + old_format_model = tf.keras.Model( + inputs, [softmax_output, argmax_output, threshold_output] + ) + + normalized_model = CharacterLevelCnnModel._normalize_model_outputs( + old_format_model, default_ind, num_labels + ) + + self.assertIsInstance(normalized_model.output, dict) + self.assertSetEqual( + set(normalized_model.output.keys()), + { + CharacterLevelCnnModel._SOFTMAX_OUTPUT, + CharacterLevelCnnModel._ARGMAX_OUTPUT, + CharacterLevelCnnModel._THRESH_OUTPUT, + }, + ) + def test_fit_and_predict_with_new_labels(self): # Initialize model cnn_model = CharacterLevelCnnModel(self.label_mapping) @@ -430,7 +462,7 @@ def test_save(self, mock_open, *mocks): StringIO.close(mock_file) @mock.patch("tensorflow.keras.Model.save", return_value=None) - @mock.patch("tensorflow.keras.models.load_model", return_value=mock.Mock()) + @mock.patch("tensorflow.keras.models.load_model", return_value=mock.MagicMock()) @mock.patch("builtins.open", side_effect=mock_open) def test_load(self, *mocks): dir = os.path.join(_resource_labeler_dir, "unstructured_model/") @@ -446,8 +478,8 @@ def test_model_construct(self): cnn_model.details() expected_layers = [ - "input_1", - "lambda", + "input_layer", + "encoding_layer", "embedding", "conv1d", "dropout", @@ -465,8 +497,9 @@ def test_model_construct(self): "dropout_4", "dense_1", "dropout_5", - "dense_2", - "thresh_arg_max_layer", + "softmax_output", + "argmax_output", + "thresh_argmax_output", ] model_layers = [layer.name for layer in cnn_model._model.layers] self.assertEqual(len(expected_layers), len(model_layers)) diff --git a/dataprofiler/tests/labelers/test_data_labelers.py b/dataprofiler/tests/labelers/test_data_labelers.py index b0cd4c7ee..e7ef03837 100644 --- a/dataprofiler/tests/labelers/test_data_labelers.py +++ b/dataprofiler/tests/labelers/test_data_labelers.py @@ -399,7 +399,7 @@ def test_has_public_functions(self, *args): @staticmethod def _setup_mock_load_model(mock_load_model): - mock_load_model.return_value = mock.Mock() + mock_load_model.return_value = mock.MagicMock() def test_load_labeler(self, mock_open, mock_load_model): diff --git a/dataprofiler/tests/plugins/test_plugins.py b/dataprofiler/tests/plugins/test_plugins.py index ec148a526..9368975dc 100644 --- a/dataprofiler/tests/plugins/test_plugins.py +++ b/dataprofiler/tests/plugins/test_plugins.py @@ -28,8 +28,8 @@ def test_plugin(): @mock.patch("dataprofiler.plugins.__init__.os.path.isdir") @mock.patch("dataprofiler.plugins.__init__.os.listdir") def test_load_plugin(self, mock_listdir, mock_isdir, mock_importlib_util): - mock_listdir.side_effect = ( - lambda folder_dir: ["__pycache__", "py"] + mock_listdir.side_effect = lambda folder_dir: ( + ["__pycache__", "py"] if folder_dir.endswith("plugins") else ["stillnotrealpy", "a.json", None] ) @@ -38,10 +38,8 @@ def test_load_plugin(self, mock_listdir, mock_isdir, mock_importlib_util): load_plugins() mock_importlib_util.spec_from_file_location.assert_not_called() - mock_listdir.side_effect = ( - lambda folder_dir: ["folder"] - if folder_dir.endswith("plugins") - else ["file.py"] + mock_listdir.side_effect = lambda folder_dir: ( + ["folder"] if folder_dir.endswith("plugins") else ["file.py"] ) mock_spec = mock.Mock() mock_importlib_util.spec_from_file_location.return_value = mock_spec diff --git a/dataprofiler/tests/profilers/test_float_column_profile.py b/dataprofiler/tests/profilers/test_float_column_profile.py index d9ec122c3..19fe4c8a6 100644 --- a/dataprofiler/tests/profilers/test_float_column_profile.py +++ b/dataprofiler/tests/profilers/test_float_column_profile.py @@ -835,7 +835,14 @@ def test_total_histogram_bin_variance(self): def test_histogram_loss(self): # run time is small - (diff_var, avg_diffvar, total_var, avg_totalvar, run_time, avg_runtime,) = ( + ( + diff_var, + avg_diffvar, + total_var, + avg_totalvar, + run_time, + avg_runtime, + ) = ( 0.3, 0.2, 0.1, @@ -855,7 +862,14 @@ def test_histogram_loss(self): self.assertEqual(expected_loss, est_loss) # run time is big - (diff_var, avg_diffvar, total_var, avg_totalvar, run_time, avg_runtime,) = ( + ( + diff_var, + avg_diffvar, + total_var, + avg_totalvar, + run_time, + avg_runtime, + ) = ( 0.3, 0.2, 0.1, diff --git a/dataprofiler/tests/profilers/test_profiler_utils.py b/dataprofiler/tests/profilers/test_profiler_utils.py index 4eee1963a..0ea0c0fc0 100644 --- a/dataprofiler/tests/profilers/test_profiler_utils.py +++ b/dataprofiler/tests/profilers/test_profiler_utils.py @@ -472,7 +472,6 @@ def test_odd_merge_profile_list(self, mock_data_labeler, *mocks): class TestAutoMultiProcessToggle(unittest.TestCase): - """ Validate profile_utils.auto_multiprocess_toggle is properly working. """ diff --git a/dataprofiler/tests/space_time_analysis/structured_space_time_analysis.py b/dataprofiler/tests/space_time_analysis/structured_space_time_analysis.py index df57854f6..5af7bc2aa 100644 --- a/dataprofiler/tests/space_time_analysis/structured_space_time_analysis.py +++ b/dataprofiler/tests/space_time_analysis/structured_space_time_analysis.py @@ -1,4 +1,5 @@ """Contains space and time analysis tests for the Dataprofiler""" + import json import os import random diff --git a/dataprofiler/tests/test_rng_utils.py b/dataprofiler/tests/test_rng_utils.py index 6ee2ed35c..8a4c4d229 100644 --- a/dataprofiler/tests/test_rng_utils.py +++ b/dataprofiler/tests/test_rng_utils.py @@ -1,4 +1,5 @@ """Validates that generator intakes DATAPROFILER_SEED properly.""" + import os import unittest import unittest.mock diff --git a/requirements-dev.txt b/requirements-dev.txt index 8c7c78684..163dae508 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ check-manifest>=0.50 black>=24.3.0 isort==5.12.0 -pre-commit==2.19.0 +pre-commit==4.3.0 tox==3.25.1 tox-conda==0.10.2 types-setuptools==67.7.0.1 diff --git a/requirements-ml.txt b/requirements-ml.txt index 31f9ca633..c8b373f22 100644 --- a/requirements-ml.txt +++ b/requirements-ml.txt @@ -1,7 +1,5 @@ scikit-learn>=0.23.2 -keras<=3.4.0 +keras>3.4.0,<4.0.0 rapidfuzz>=2.6.1 -tensorflow>=2.16.0; sys.platform != 'darwin' -tensorflow>=2.16.0; sys_platform == 'darwin' and platform_machine != 'arm64' -tensorflow-macos>=2.16.0; sys_platform == 'darwin' and platform_machine == 'arm64' +tensorflow>=2.16.0 tqdm>=4.0.0 diff --git a/requirements-test.txt b/requirements-test.txt index 725b23849..cf127b607 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -6,4 +6,4 @@ pytest-cov>=2.8.1 pytest-xdist>=2.1.0 pytest-forked>=1.3.0 toolz>=0.10.0 -memray>=1.7.0,<1.12.0 +memray>=1.18.0 diff --git a/requirements.txt b/requirements.txt index 1036c4331..3db3daad4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ h5py>=2.10.0 wheel>=0.33.1 -numpy<2.0.0 +numpy>=1.22.0,<3.0.0 pandas>=1.1.2,<3.0.0 python-dateutil>=2.7.5 pytz>=2020.1